diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index f87ab9e9564..a63b1e3a983 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -1,7 +1,7 @@ import { setImmediate as immediate } from "node:timers/promises"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { expect, it, vi } from "vitest"; import { agents, budgetPolicies, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db"; import { heartbeatService } from "../services/heartbeat.js"; @@ -113,7 +113,7 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => const [run] = await state.db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", finishedAt: new Date(), contextSnapshot: { issueId: task.id } }).returning({ id: heartbeatRuns.id }); const backlog = async (count: number) => (await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ - companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id, + companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id, createdAt: new Date(), updatedAt: new Date(), }))).returning({ id: issues.id })).map(row => row.id); return { runId: run.id, taskId: task.id, siblingId: sibling.id, backlog, affected: [parent.id, task.id, dependent.id].sort() }; } @@ -223,6 +223,68 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => expect(checkedAt.get(later)! - eventAt).toBeLessThanOrEqual(5_000); await quiet(); }); + async function advanceWithoutWaiting(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); await immediate(); } + } + it("keeps a saved deadline of a task that a changed-tasks backstop pass does not load", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + const [target] = await graph.backlog(20); + const checkedAt: number[] = []; + await recordedRecovery({ stranded: async scope => { + if (scope.issueId !== target) return; + checkedAt.push(Date.now()); + scope.onDeferredDeadline(Date.now() + 16 * 60_000); + } }); + await settleWork({ backstopPending: true }); + for (let round = 0; round < 2; round++) { + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + } + const savedFor = checkedAt.at(-1)! + 16 * 60_000; + checkedAt.length = 0; + await vi.advanceTimersByTimeAsync(savedFor - Date.now() + 2_000); + await settleWork({ backstopPending: true }); + expect(checkedAt.length).toBeGreaterThan(0); + expect(checkedAt[0]! - savedFor).toBeLessThanOrEqual(5_000); + }); + it("starts the next changed-tasks window from the start of a slow backstop scan", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + const backlog = (await graph.backlog(250)).sort(); + const behind = backlog[0]!; + let slow = false, changedBehind = false; + const checkedAt = new Map(); + await recordedRecovery({ stranded: async scope => { + checkedAt.set(scope.issueId, [...checkedAt.get(scope.issueId) ?? [], Date.now()]); + if (!slow) return; + vi.setSystemTime(Date.now() + 20_000); + if (scope.issueId === behind && !changedBehind) { + changedBehind = true; + await state.db.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, behind)); + } + } }, { fullBackstopMs: 24 * 60 * 60_000 }); + await settleWork({ backstopPending: true }); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + await state.db.update(issues).set({ updatedAt: new Date() }).where(inArray(issues.id, backlog)); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + slow = true; + checkedAt.clear(); + const scanStartedAt = Date.now(); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + slow = false; + expect(changedBehind).toBe(true); + expect(backlog.every(id => checkedAt.has(id))).toBe(true); + expect(Date.now() - scanStartedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS); + checkedAt.clear(); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS + 60_000); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + expect(checkedAt.has(behind)).toBe(true); + }, 300_000); it("keeps the company pass for hold releases that can admit any task", async () => { const f = await fixture(); const graph = await runGraph(f.companyId, f.agentId); diff --git a/server/src/__tests__/issue-recovery-idle-backstop.test.ts b/server/src/__tests__/issue-recovery-idle-backstop.test.ts new file mode 100644 index 00000000000..baf8fd03798 --- /dev/null +++ b/server/src/__tests__/issue-recovery-idle-backstop.test.ts @@ -0,0 +1,166 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq, sql } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, heartbeatRuns, issues } from "@paperclipai/db"; +import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; +import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, notifyIssueGraphRecovery } + from "../services/issue-recovery-notifications.js"; +import { issueService } from "../services/issues.js"; +import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; +import { promoteDeferredWorkIntents } from "../services/work-intent-outbox.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; +import { seedRecoveryPopulation } from "./helpers/recovery-population.js"; + +vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1"; }); + +const MINUTE = 60_000; +type Recorded = { at: number; name: string; fields: Record }; + +describeIssueRecovery("stranded-task checks at idle for a staging-shaped company", () => { + const state = issueRecoveryFixture({ meter: true }); + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); + await Promise.allSettled([...state.passes]); await immediate(); + } + } + + function record() { + const events: Recorded[] = []; + vi.spyOn(workSignalDebug, "event").mockImplementation((topic, name, fields = {}) => { + if (topic === "stranded-assignments") events.push({ at: Date.now(), name, fields }); + }); + vi.spyOn(workSignalDebug, "signal").mockImplementation(() => {}); + return events; + } + + function summarize(events: Recorded[]) { + const checks: Record = {}, loads: Record = {}; + for (const event of events) { + if (event.name === "check") checks[String(event.fields.wake)] = (checks[String(event.fields.wake)] ?? 0) + 1; + if (event.name === "load") { + const scope = String(event.fields.scope); + const kind = scope === "*" ? "reload" : scope.endsWith(",true]") ? "backstop" : /^\["[^"]+",null,null,null,null\]$/.test(scope) + ? "company" : "hint"; + loads[kind] = (loads[kind] ?? 0) + 1; + } + } + return { checks, loads, totalChecks: Object.values(checks).reduce((sum, count) => sum + count, 0) }; + } + + const checkedAt = (events: Recorded[], issueId: string) => + events.find(event => event.name === "check" && event.fields.issueId === issueId)?.at ?? null; + + async function seed() { + const f = await state.fixture(); + const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 50, inReview: 780, inProgress: 40, todo: 30, + recoveryActions: 12, terminated: 3 }); + const [side] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Finished side task", status: "done" }) + .returning({ id: issues.id }); + const activity = () => state.db.transaction(tx => notifyIssueGraphRecovery(tx, { companyId: f.companyId, issueId: side!.id })); + return { ...f, population, activity }; + } + + async function boot() { + const f = await seed(); + const events = record(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await state.start({}); + await advance(10 * MINUTE); + return { ...f, events, boot: summarize(events.splice(0)) }; + } + + async function heldTask(f: Awaited>) { + const old = new Date(Date.now() - 2 * 60 * MINUTE); + const [held] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Held worker", role: "engineer", status: "paused", + lifecycleHolds: ["manual"], adapterType: "process", reportsTo: f.population.agentIds[1], createdAt: old, updatedAt: old, + adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } } }).returning({ id: agents.id }); + const [waiting] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Waits for its agent", status: "todo", + assigneeAgentId: held!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id }); + return { held: held!, waiting: waiting! }; + } + + async function trickle(f: Awaited>, minutes: number) { + const before = state.meter.read(); + for (let minute = 0; minute < minutes; minute++) { await f.activity(); await advance(MINUTE); } + const after = state.meter.read(); + const raw = f.events.splice(0); + return { ...summarize(raw), raw, statements: after.statements - before.statements, + bytes: after.bytesReceived - before.bytesReceived }; + } + + it("checks only changed tasks between hourly full passes while unrelated activity continues", async () => { + const f = await boot(); + const between = await trickle(f, 30); + const hour = await trickle(f, 30); + const { raw: _first, ...firstHalfHour } = between, { raw: _second, ...secondHalfHour } = hour; + console.info(JSON.stringify({ boot: f.boot, firstHalfHour, secondHalfHour, + perSecond: { checks: (between.totalChecks + hour.totalChecks) / 3600, statements: (between.statements + hour.statements) / 3600 } })); + expect(f.boot.totalChecks).toBeGreaterThanOrEqual(f.population.inReview.length); + expect(between.totalChecks).toBe(0); + expect(between.statements).toBeLessThan(2_000); + expect(hour.checks.backstop ?? 0).toBeLessThanOrEqual(f.population.inReview.length + 100); + }, 1_800_000); + + it("recovers a task whose hint was never sent within the backstop bound", async () => { + const f = await boot(); + const worker = f.population.agentIds[7]!; + const now = new Date(); + const [missed] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Created without a hint", status: "todo", + assigneeAgentId: worker, createdAt: now, updatedAt: now }).returning({ id: issues.id }); + const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_BACKSTOP_MS + MINUTE) / MINUTE) + 1); + const checked = checkedAt(window.raw, missed!.id); + expect(checked).not.toBeNull(); + expect(checked! - now.getTime()).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE); + const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${missed!.id}`)); + expect(runs.length).toBeGreaterThan(0); + }, 1_800_000); + + it("re-checks a task whose only change is outside its own rows within the hourly full pass", async () => { + const f = await boot(); + const { held, waiting } = await heldTask(f); + await state.db.execute(sql`update agents set status = 'idle', lifecycle_holds = '[]'::jsonb where id = ${held.id}`); + const changedAt = Date.now(); + const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE) / MINUTE)); + const checks = window.raw.filter(event => event.name === "check" && event.fields.issueId === waiting.id); + expect(checks.length).toBeGreaterThan(0); + expect(checks[0]!.at - changedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS); + expect(checks[0]!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE); + const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${waiting.id}`)); + expect(runs.length).toBeGreaterThan(0); + }, 1_800_000); + + it("picks up a finished run, a user message, a raised budget and a resumed agent within a minute", async () => { + const f = await boot(); + await trickle(f, 6); + const [target, messaged] = f.population.inReview; + const within = async (issueId: string, act: () => Promise) => { + f.events.length = 0; + const at = Date.now(); + await act(); + await advance(60_000); + const checked = checkedAt(f.events, issueId); + expect(checked, issueId).not.toBeNull(); + expect(checked! - at).toBeLessThanOrEqual(60_000); + }; + const [run] = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${target!}`).limit(1); + await within(target!, () => state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: run!.id }))); + await advance(31_000); + await within(messaged!, () => issueService(state.db).addComment(messaged!, "Please look again", { userId: "recovery-user" })); + await advance(31_000); + await within(target!, () => state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" }))); + await advance(31_000); + const { held, waiting } = await heldTask(f); + await advance(31_000); + await within(waiting.id, async () => { + await state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }).where(eq(agents.id, held.id)); + await state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" })); + }); + }, 1_800_000); +}); diff --git a/server/src/__tests__/issue-recovery-work-debug-attribution.test.ts b/server/src/__tests__/issue-recovery-work-debug-attribution.test.ts index 8b0282ac0e5..eceba1544ee 100644 --- a/server/src/__tests__/issue-recovery-work-debug-attribution.test.ts +++ b/server/src/__tests__/issue-recovery-work-debug-attribution.test.ts @@ -39,7 +39,7 @@ describeIssueRecovery("issue recovery work signal attribution", () => { expect(stranded).toContainEqual(["stranded-assignments", "load", { scope: JSON.stringify([f.companyId, null, run.id, null, null]) }]); expect(stranded).toContainEqual(["stranded-assignments", "check", - expect.objectContaining({ issueId: task.id, companyId: f.companyId, wake: "load" })]); + expect.objectContaining({ issueId: task.id, companyId: f.companyId, wake: expect.stringMatching(/^(hint|backstop)$/) })]); } finally { enabled.mockRestore(); event.mockRestore(); signal.mockRestore(); } }); diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index 08751b36220..5c5038d5425 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -10,6 +10,7 @@ export type IssueRecoveryScope = { companyId: string; issueId?: string; graphRun export const ISSUE_RECOVERY_MAX_ISSUE_IDS = 100; export const ISSUE_RECOVERY_BACKSTOP_MS = 5 * 60_000; +export const ISSUE_RECOVERY_FULL_BACKSTOP_MS = 60 * 60_000; const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; const COMPANY_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.taskWatchdogs]; diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 3315703383e..1983df05181 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -20,6 +20,7 @@ import { import { logger } from "../../middleware/logger.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { evaluateAgentInvokability } from "../agent-invokability.js"; +import { changedTaskCondition } from "./changed-tasks.js"; import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js"; import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; @@ -511,7 +512,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { order by lr.created_at desc limit 1), false)`; } - async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null }, + async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null; + changedSince?: Date }, cursor: string | null, limit: number) { const at = now(); const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} @@ -531,6 +533,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { scope.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? sql`${issues.id} > ${cursor}` : undefined, scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined, + scope.changedSince ? changedTaskCondition(scope.changedSince) : undefined, inArray(issues.status, [...OPEN_STATUSES]), sql`${issues.assigneeAgentId} is not null`, isNull(issues.assigneeUserId), diff --git a/server/src/services/recovery/changed-tasks.ts b/server/src/services/recovery/changed-tasks.ts new file mode 100644 index 00000000000..2a0832131aa --- /dev/null +++ b/server/src/services/recovery/changed-tasks.ts @@ -0,0 +1,18 @@ +import { sql } from "drizzle-orm"; +import { issues } from "@paperclipai/db"; + +export function changedTaskCondition(since: Date) { + const at = sql`${since.toISOString()}::timestamptz`; + return sql`(${issues.updatedAt} >= ${at} + or exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} + and (r.context_snapshot->>'issueId') = ${issues.id}::text and r.updated_at >= ${at}) + or exists (select 1 from issue_recovery_actions a where a.company_id = ${issues.companyId} + and a.source_issue_id = ${issues.id} and a.updated_at >= ${at}) + or exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId} + and (w.payload->>'issueId') = ${issues.id}::text and w.updated_at >= ${at}) + or exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId} + and t.issue_id = ${issues.id} and t.updated_at >= ${at}) + or exists (select 1 from issue_comments c where c.company_id = ${issues.companyId} + and c.issue_id = ${issues.id} and c.created_at >= ${at}) + or exists (select 1 from agents ag where ag.id = ${issues.assigneeAgentId} and ag.updated_at >= ${at}))`; +} diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 6a8115737ed..9926f1c9fb8 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -3,14 +3,15 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; -import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } +import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } from "../issue-recovery-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { workSignalDebug } from "@paperclipai/db/work-signal-debug"; import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js"; +import { changedTaskCondition } from "./changed-tasks.js"; import { createRecoveryPassReads, type RecoveryPassReads } from "./pass-reads.js"; -type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true }; +type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true; origin?: "reload" | "backstop" | "company" | "hint" }; type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void; pass?: RecoveryPassReads }; const PAGE_SIZE = 100; @@ -35,22 +36,39 @@ export function registerIssueRecoveryWork( schedulingSuppressed(): Promise; /** The assigned-work invariant (assigned-work-sweep.ts). */ assignedWork?: { - listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[] }, cursor: string | null, limit: number): + listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; changedSince?: Date }, cursor: string | null, limit: number): Promise>; checkIssue(scope: Scope): Promise; }; }, canRun: () => boolean, - options: { backstopMs?: number } = {}, + options: { backstopMs?: number; fullBackstopMs?: number } = {}, ) { + const backstopMs = options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS; + const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS; function fullPasses() { - const completedAt = new Map(); + const completedAt = new Map(), changedAt = new Map(); + const scans = new Map(); + const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0); + const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0); return { - backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop - ? Math.max(completedAt.get("*") ?? 0, completedAt.get(scope.companyId) ?? 0) + (options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS) - : null, - loaded(scope: IssueRecoveryScope | undefined, complete: boolean) { - if (complete && !scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now()); + backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null, + changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null { + if (firstPage || !scans.has(key)) scans.set(key, { startedAt: Date.now(), + since: !scope?.backstop || Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null + : new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS) }); + return scans.get(key)!.since; + }, + loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) { + if (!complete) return; + const scan = scans.get(key); + scans.delete(key); + const startedAt = scan?.startedAt ?? Date.now(); + if (scan?.since && scope) changedAt.set(scope.companyId, Math.max(changedAt.get(scope.companyId) ?? 0, startedAt)); + else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) { + const companyKey = scope?.companyId ?? "*"; + completedAt.set(companyKey, Math.max(completedAt.get(companyKey) ?? 0, startedAt)); + } }, }; } @@ -98,8 +116,10 @@ export function registerIssueRecoveryWork( if (!enabled) { cursors.delete(key); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; const cursor = cursors.get(key); + const changedSince = passes.changedSince(scope, key, !cursor); const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( scope ? eq(issues.companyId, scope.companyId) : undefined, + changedSince ? changedTaskCondition(changedSince) : undefined, scope?.issueId ? eq(issues.id, scope.issueId) : undefined, scope?.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? gt(issues.id, cursor) : undefined, @@ -111,13 +131,16 @@ export function registerIssueRecoveryWork( )).orderBy(asc(issues.id)).limit(PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); - passes.loaded(scope, complete); - return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + passes.loaded(scope, key, complete); + const keepUnseen = changedSince ? { keepUnseen: true as const } : {}; + const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const + : scope.issueId || isGraph(scope) ? "hint" as const : "company" as const; + return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete, ...keepUnseen }; }, check: async entry => { if (!enabled) return null; if (workSignalDebug.enabled) workSignalDebug.event(queue, "check", { issueId: entry.id, companyId: entry.companyId, - wake: entry.fromDeadline ? "deadline" : "load", dueAt: entry.dueAt }); + wake: entry.fromDeadline ? "deadline" : entry.origin ?? "load", dueAt: entry.dueAt }); let next: number | null = null; await check({ companyId: entry.companyId, issueId: entry.id, pass, onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); @@ -149,13 +172,14 @@ export function registerIssueRecoveryWork( const key = scopeKey(scope); if (!enabled) { cursors.delete(key); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; + const changedSince = passes.changedSince(scope, key, !cursors.has(key)); const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId, - issueIds: scope.issueIds } : {}, cursors.get(key) ?? null, PAGE_SIZE); + issueIds: scope.issueIds, ...(changedSince ? { changedSince } : {}) } : {}, cursors.get(key) ?? null, PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); - passes.loaded(scope, complete); + passes.loaded(scope, key, complete); return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(), - ...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete }; + ...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete, ...(changedSince ? { keepUnseen: true as const } : {}) }; }, check: async entry => { if (!enabled) return null; diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index 232b816e484..647f75d9c57 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -37,7 +37,7 @@ export function registerScopedDeadlineWork( * the scopes of its own key (and unkeyed scopes), not against every scope. */ scopeKey?: { entry(entry: Entry): string; scope(scope: Scope): string | undefined }; - load(scope?: Scope, signal?: AbortSignal): Promise; + load(scope?: Scope, signal?: AbortSignal): Promise; // Owners may retain pending queue priority while refreshing eligibility. mergeEntry?(pending: Entry, loaded: Entry): Entry; priority?(entry: Entry): number; @@ -165,7 +165,8 @@ export function registerScopedDeadlineWork( const scan = scans.get(hint) ?? { version: versions.get(hint) ?? 0, seen: new Set() }; scans.set(hint, scan); const loaded = await options.load(scope, signal); - const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded; + const result: { rows: Entry[] | null; complete: boolean; keepUnseen?: boolean } = Array.isArray(loaded) + ? { rows: loaded, complete: true } : loaded; const progressScopes = [...scans].map(([other, progress]) => ({ progress, scope: decode(other) })); for (const entry of result.rows ?? []) { const pending = entries.get(entry.id); @@ -174,7 +175,7 @@ export function registerScopedDeadlineWork( for (const { progress, scope: other } of progressScopes) if (options.matches(entry, other)) progress.seen.add(entry.id); } if (!result.complete) { dirty.add(hint); continue; } - if (result.rows !== null) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { + if (result.rows !== null && !result.keepUnseen) for (const entry of entriesIn(scope)) if (options.matches(entry, scope) && !scan.seen.has(entry.id)) { entries.delete(entry.id); failures.delete(entry.id); } scans.delete(hint);