perf(recovery): re-check only changed tasks between hourly full backstop passes

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-15 00:14:26 -06:00
1 parent 6400698a8e
commit 379dda9e26
8 files changed
+284 -19

No files matched your search

@@ -0,0 +1,54 @@
import { agents, heartbeatRuns, issueRecoveryActions, issues, type Db } from "@paperclipai/db";
const HOUR = 60 * 60_000;
export type RecoveryPopulation = {
agentIds: string[];
pausedAgentId: string;
terminatedAgentId: string;
inReview: string[];
inProgress: string[];
todo: string[];
withRecoveryAction: string[];
terminated: string[];
};
export async function seedRecoveryPopulation(db: Db, companyId: string, counts: {
agents: number; inReview: number; inProgress?: number; todo?: number; recoveryActions?: number; terminated?: number;
}): Promise<RecoveryPopulation> {
const config = { adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() },
runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } }, role: "engineer" };
const [ceo] = await db.insert(agents).values({ ...config, companyId, name: "Chief", status: "idle" }).returning({ id: agents.id });
const managers = await db.insert(agents).values(Array.from({ length: 6 }, (_, index) => ({ ...config, companyId,
name: `Manager ${index}`, status: "idle", reportsTo: ceo!.id }))).returning({ id: agents.id });
const workers = await db.insert(agents).values(Array.from({ length: Math.max(1, counts.agents - 9) }, (_, index) => ({ ...config,
companyId, name: `Worker ${index}`, status: "idle", reportsTo: managers[index % managers.length]!.id }))).returning({ id: agents.id });
const [paused] = await db.insert(agents).values({ ...config, companyId, name: "Paused worker", status: "paused",
lifecycleHolds: ["manual"], reportsTo: managers[0]!.id }).returning({ id: agents.id });
const [terminatedAgent] = await db.insert(agents).values({ ...config, companyId, name: "Former worker", status: "terminated",
reportsTo: managers[1]!.id }).returning({ id: agents.id });
const workerIds = workers.map(row => row.id);
const finishedAt = new Date(Date.now() - HOUR);
const insertTasks = async (count: number, values: (index: number) => Partial<typeof issues.$inferInsert>) => count === 0 ? []
: (await db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ companyId, title: `Task ${index}`,
...values(index) }))).returning({ id: issues.id, assigneeAgentId: issues.assigneeAgentId })).map(row => row);
const reviews = await insertTasks(counts.inReview, index => ({ status: "in_review",
assigneeAgentId: workerIds[index % workerIds.length] }));
if (reviews.length) await db.insert(heartbeatRuns).values(reviews.map(task => ({ companyId, agentId: task.assigneeAgentId!,
status: "succeeded", runtimeMode: "native", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } })));
const progressing = await insertTasks(counts.inProgress ?? 0, () => ({ status: "in_progress", assigneeAgentId: paused!.id }));
if (progressing.length) await db.insert(heartbeatRuns).values(progressing.map(task => ({ companyId, agentId: paused!.id,
status: "cancelled", errorCode: "agent_paused", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } })));
const todo = await insertTasks(counts.todo ?? 0, () => ({ status: "todo", assigneeAgentId: paused!.id }));
const terminated = await insertTasks(counts.terminated ?? 0, () => ({ status: "todo", assigneeAgentId: terminatedAgent!.id }));
const withAction = reviews.slice(0, counts.recoveryActions ?? 0);
if (withAction.length) await db.insert(issueRecoveryActions).values(withAction.map((task, index) => ({ companyId,
sourceIssueId: task.id, kind: "missing_disposition", status: "active", ownerType: "agent", ownerAgentId: task.assigneeAgentId,
cause: "seeded", fingerprint: `seeded-${index}`, nextAction: "seeded" })));
return { agentIds: [ceo!.id, ...managers.map(row => row.id), ...workerIds, paused!.id, terminatedAgent!.id],
pausedAgentId: paused!.id, terminatedAgentId: terminatedAgent!.id,
inReview: reviews.map(row => row.id), inProgress: progressing.map(row => row.id), todo: todo.map(row => row.id),
withRecoveryAction: withAction.map(row => row.id), terminated: terminated.map(row => row.id) };
}
@@ -113,7 +113,7 @@ describeIssueRecovery("issue recovery production producers and deadlines", () =>
const [run] = await state.db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", finishedAt: new Date(),
contextSnapshot: { issueId: task.id } }).returning({ id: heartbeatRuns.id });
const backlog = async (count: number) => (await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({
companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id,
companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id, createdAt: new Date(), updatedAt: new Date(),
}))).returning({ id: issues.id })).map(row => row.id);
return { runId: run.id, taskId: task.id, siblingId: sibling.id, backlog, affected: [parent.id, task.id, dependent.id].sort() };
}
@@ -0,0 +1,166 @@
import { setImmediate as immediate } from "node:timers/promises";
import { and, eq, sql } from "drizzle-orm";
import { expect, it, vi } from "vitest";
import { agents, heartbeatRuns, issues } from "@paperclipai/db";
import { workSignalDebug } from "@paperclipai/db/work-signal-debug";
import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, notifyIssueGraphRecovery }
from "../services/issue-recovery-notifications.js";
import { issueService } from "../services/issues.js";
import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js";
import { promoteDeferredWorkIntents } from "../services/work-intent-outbox.js";
import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js";
import { seedRecoveryPopulation } from "./helpers/recovery-population.js";
vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1"; });
const MINUTE = 60_000;
type Recorded = { at: number; name: string; fields: Record<string, unknown> };
describeIssueRecovery("stranded-task checks at idle for a staging-shaped company", () => {
const state = issueRecoveryFixture({ meter: true });
async function advance(ms: number) {
const until = Date.now() + ms;
while (Date.now() < until) {
await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now()));
await Promise.allSettled([...state.passes]); await immediate();
}
}
function record() {
const events: Recorded[] = [];
vi.spyOn(workSignalDebug, "event").mockImplementation((topic, name, fields = {}) => {
if (topic === "stranded-assignments") events.push({ at: Date.now(), name, fields });
});
vi.spyOn(workSignalDebug, "signal").mockImplementation(() => {});
return events;
}
function summarize(events: Recorded[]) {
const checks: Record<string, number> = {}, loads: Record<string, number> = {};
for (const event of events) {
if (event.name === "check") checks[String(event.fields.wake)] = (checks[String(event.fields.wake)] ?? 0) + 1;
if (event.name === "load") {
const scope = String(event.fields.scope);
const kind = scope === "*" ? "reload" : scope.endsWith(",true]") ? "backstop" : /^\["[^"]+",null,null,null,null\]$/.test(scope)
? "company" : "hint";
loads[kind] = (loads[kind] ?? 0) + 1;
}
}
return { checks, loads, totalChecks: Object.values(checks).reduce((sum, count) => sum + count, 0) };
}
const checkedAt = (events: Recorded[], issueId: string) =>
events.find(event => event.name === "check" && event.fields.issueId === issueId)?.at ?? null;
async function seed() {
const f = await state.fixture();
const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 50, inReview: 780, inProgress: 40, todo: 30,
recoveryActions: 12, terminated: 3 });
const [side] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Finished side task", status: "done" })
.returning({ id: issues.id });
const activity = () => state.db.transaction(tx => notifyIssueGraphRecovery(tx, { companyId: f.companyId, issueId: side!.id }));
return { ...f, population, activity };
}
async function boot() {
const f = await seed();
const events = record();
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
await state.start({});
await advance(10 * MINUTE);
return { ...f, events, boot: summarize(events.splice(0)) };
}
async function heldTask(f: Awaited<ReturnType<typeof boot>>) {
const old = new Date(Date.now() - 2 * 60 * MINUTE);
const [held] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Held worker", role: "engineer", status: "paused",
lifecycleHolds: ["manual"], adapterType: "process", reportsTo: f.population.agentIds[1], createdAt: old, updatedAt: old,
adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() },
runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } } }).returning({ id: agents.id });
const [waiting] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Waits for its agent", status: "todo",
assigneeAgentId: held!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id });
return { held: held!, waiting: waiting! };
}
async function trickle(f: Awaited<ReturnType<typeof boot>>, minutes: number) {
const before = state.meter.read();
for (let minute = 0; minute < minutes; minute++) { await f.activity(); await advance(MINUTE); }
const after = state.meter.read();
const raw = f.events.splice(0);
return { ...summarize(raw), raw, statements: after.statements - before.statements,
bytes: after.bytesReceived - before.bytesReceived };
}
it("checks only changed tasks between hourly full passes while unrelated activity continues", async () => {
const f = await boot();
const between = await trickle(f, 30);
const hour = await trickle(f, 30);
const { raw: _first, ...firstHalfHour } = between, { raw: _second, ...secondHalfHour } = hour;
console.info(JSON.stringify({ boot: f.boot, firstHalfHour, secondHalfHour,
perSecond: { checks: (between.totalChecks + hour.totalChecks) / 3600, statements: (between.statements + hour.statements) / 3600 } }));
expect(f.boot.totalChecks).toBeGreaterThanOrEqual(f.population.inReview.length);
expect(between.totalChecks).toBe(0);
expect(between.statements).toBeLessThan(2_000);
expect(hour.checks.backstop ?? 0).toBeLessThanOrEqual(f.population.inReview.length + 100);
}, 1_800_000);
it("recovers a task whose hint was never sent within the backstop bound", async () => {
const f = await boot();
const worker = f.population.agentIds[7]!;
const now = new Date();
const [missed] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Created without a hint", status: "todo",
assigneeAgentId: worker, createdAt: now, updatedAt: now }).returning({ id: issues.id });
const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_BACKSTOP_MS + MINUTE) / MINUTE) + 1);
const checked = checkedAt(window.raw, missed!.id);
expect(checked).not.toBeNull();
expect(checked! - now.getTime()).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE);
const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${missed!.id}`));
expect(runs.length).toBeGreaterThan(0);
}, 1_800_000);
it("re-checks a task whose only change is outside its own rows within the hourly full pass", async () => {
const f = await boot();
const { held, waiting } = await heldTask(f);
await state.db.execute(sql`update agents set status = 'idle', lifecycle_holds = '[]'::jsonb where id = ${held.id}`);
const changedAt = Date.now();
const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE) / MINUTE));
const checks = window.raw.filter(event => event.name === "check" && event.fields.issueId === waiting.id);
expect(checks.length).toBeGreaterThan(0);
expect(checks[0]!.at - changedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS);
expect(checks[0]!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE);
const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
.where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${waiting.id}`));
expect(runs.length).toBeGreaterThan(0);
}, 1_800_000);
it("picks up a finished run, a user message, a raised budget and a resumed agent within a minute", async () => {
const f = await boot();
await trickle(f, 6);
const [target, messaged] = f.population.inReview;
const within = async (issueId: string, act: () => Promise<unknown>) => {
f.events.length = 0;
const at = Date.now();
await act();
await advance(60_000);
const checked = checkedAt(f.events, issueId);
expect(checked, issueId).not.toBeNull();
expect(checked! - at).toBeLessThanOrEqual(60_000);
};
const [run] = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
.where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${target!}`).limit(1);
await within(target!, () => state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: run!.id })));
await advance(31_000);
await within(messaged!, () => issueService(state.db).addComment(messaged!, "Please look again", { userId: "recovery-user" }));
await advance(31_000);
await within(target!, () => state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" })));
await advance(31_000);
const { held, waiting } = await heldTask(f);
await advance(31_000);
await within(waiting.id, async () => {
await state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }).where(eq(agents.id, held.id));
await state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" }));
});
}, 1_800_000);
});
@@ -11,6 +11,7 @@ import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js
import { instanceSettingsService } from "../services/instance-settings.js";
import { issueService } from "../services/issues.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { meterDatabaseStatements } from "./helpers/database-statement-meter.js";
vi.setConfig({ testTimeout: 90_000 });
@@ -18,7 +19,8 @@ const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_D
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
export const describeIssueRecovery: ReturnType<typeof describe.skipIf> = describe.skipIf(!support.supported);
export function issueRecoveryFixture() {
export function issueRecoveryFixture(options: { meter?: boolean } = {}) {
let meter: ReturnType<typeof meterDatabaseStatements> | undefined;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: ReturnType<typeof createDb>;
let heartbeat: ReturnType<typeof heartbeatService>;
@@ -35,6 +37,7 @@ export function issueRecoveryFixture() {
beforeAll(async () => {
if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
if (options.meter) meter = meterDatabaseStatements(db, external ?? database!.connectionString);
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
}, 90_000);
beforeEach(async () => {
@@ -49,7 +52,7 @@ export function issueRecoveryFixture() {
vi.useRealTimers(); vi.restoreAllMocks();
expect(errors).toEqual([]);
});
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
afterAll(async () => { meter?.uninstall(); await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
async function start(options: { backstopMs?: number } = { backstopMs: 0 }) {
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
onError: error => errors.push(error) });
@@ -232,6 +235,7 @@ export function issueRecoveryFixture() {
return {
start, compete, fixture, completingProvider, settled, quiet, finish, completed, savedConversation,
get db() { return db; },
get meter() { return meter!; },
get heartbeat() { return heartbeat; },
set heartbeat(value: typeof heartbeat) { heartbeat = value; },
get coordinator() { return coordinator; },
@@ -10,6 +10,7 @@ export type IssueRecoveryScope = { companyId: string; issueId?: string; graphRun
export const ISSUE_RECOVERY_MAX_ISSUE_IDS = 100;
export const ISSUE_RECOVERY_BACKSTOP_MS = 5 * 60_000;
export const ISSUE_RECOVERY_FULL_BACKSTOP_MS = 60 * 60_000;
const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$";
const COMPANY_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.taskWatchdogs];
@@ -20,6 +20,7 @@ import {
import { logger } from "../../middleware/logger.js";
import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
import { evaluateAgentInvokability } from "../agent-invokability.js";
import { changedTaskCondition } from "./changed-tasks.js";
import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js";
import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js";
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
@@ -510,7 +511,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
order by lr.created_at desc limit 1), false)`;
}
async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null },
async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null;
changedSince?: Date },
cursor: string | null, limit: number) {
const at = now();
const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId}
@@ -530,6 +532,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
scope.issueIds ? inArray(issues.id, scope.issueIds) : undefined,
cursor ? sql`${issues.id} > ${cursor}` : undefined,
scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined,
scope.changedSince ? changedTaskCondition(scope.changedSince) : undefined,
inArray(issues.status, [...OPEN_STATUSES]),
sql`${issues.assigneeAgentId} is not null`,
isNull(issues.assigneeUserId),
@@ -0,0 +1,18 @@
import { sql } from "drizzle-orm";
import { issues } from "@paperclipai/db";
export function changedTaskCondition(since: Date) {
const at = sql`${since.toISOString()}::timestamptz`;
return sql`(${issues.updatedAt} >= ${at}
or exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId}
and (r.context_snapshot->>'issueId') = ${issues.id}::text and r.updated_at >= ${at})
or exists (select 1 from issue_recovery_actions a where a.company_id = ${issues.companyId}
and a.source_issue_id = ${issues.id} and a.updated_at >= ${at})
or exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId}
and (w.payload->>'issueId') = ${issues.id}::text and w.updated_at >= ${at})
or exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId}
and t.issue_id = ${issues.id} and t.updated_at >= ${at})
or exists (select 1 from issue_comments c where c.company_id = ${issues.companyId}
and c.issue_id = ${issues.id} and c.created_at >= ${at})
or exists (select 1 from agents ag where ag.id = ${issues.assigneeAgentId} and ag.updated_at >= ${at}))`;
}
+34 -15
View File
@@ -3,13 +3,14 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope }
import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope }
from "../issue-recovery-notifications.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
import { workSignalDebug } from "@paperclipai/db/work-signal-debug";
import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js";
import { changedTaskCondition } from "./changed-tasks.js";
type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true };
type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true; origin?: "reload" | "backstop" | "company" | "hint" };
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void };
const PAGE_SIZE = 100;
@@ -33,22 +34,35 @@ export function registerIssueRecoveryWork(
schedulingSuppressed(): Promise<boolean>;
/** The assigned-work invariant (assigned-work-sweep.ts). */
assignedWork?: {
listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[] }, cursor: string | null, limit: number):
listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; changedSince?: Date }, cursor: string | null, limit: number):
Promise<Array<{ id: string; companyId: string; updatedAt: Date }>>;
checkIssue(scope: Scope): Promise<unknown>;
};
},
canRun: () => boolean,
options: { backstopMs?: number } = {},
options: { backstopMs?: number; fullBackstopMs?: number } = {},
) {
const backstopMs = options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS;
const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS;
function fullPasses() {
const completedAt = new Map<string, number>();
const completedAt = new Map<string, number>(), changedAt = new Map<string, number>();
const scans = new Map<string, Date | null>();
const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0);
const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0);
return {
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop
? Math.max(completedAt.get("*") ?? 0, completedAt.get(scope.companyId) ?? 0) + (options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS)
: null,
loaded(scope: IssueRecoveryScope | undefined, complete: boolean) {
if (complete && !scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now());
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null,
changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null {
if (!scope?.backstop) return null;
if (firstPage || !scans.has(key)) scans.set(key, Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS));
return scans.get(key) ?? null;
},
loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) {
if (!complete) return;
const changed = scans.get(key) ?? null;
scans.delete(key);
if (changed && scope) changedAt.set(scope.companyId, Date.now());
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now());
},
};
}
@@ -94,8 +108,10 @@ export function registerIssueRecoveryWork(
if (!enabled) { cursors.delete(key); return []; }
if (await toCompany(scope, worker)) return { rows: null, complete: true };
const cursor = cursors.get(key);
const changedSince = passes.changedSince(scope, key, !cursor);
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
scope ? eq(issues.companyId, scope.companyId) : undefined,
changedSince ? changedTaskCondition(changedSince) : undefined,
scope?.issueId ? eq(issues.id, scope.issueId) : undefined,
scope?.issueIds ? inArray(issues.id, scope.issueIds) : undefined,
cursor ? gt(issues.id, cursor) : undefined,
@@ -107,13 +123,15 @@ export function registerIssueRecoveryWork(
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
passes.loaded(scope, complete);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
passes.loaded(scope, key, complete);
const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const
: scope.issueId || isGraph(scope) ? "hint" as const : "company" as const;
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete };
},
check: async entry => {
if (!enabled) return null;
if (workSignalDebug.enabled) workSignalDebug.event(queue, "check", { issueId: entry.id, companyId: entry.companyId,
wake: entry.fromDeadline ? "deadline" : "load", dueAt: entry.dueAt });
wake: entry.fromDeadline ? "deadline" : entry.origin ?? "load", dueAt: entry.dueAt });
let next: number | null = null;
await check({ companyId: entry.companyId, issueId: entry.id,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
@@ -145,11 +163,12 @@ export function registerIssueRecoveryWork(
const key = scopeKey(scope);
if (!enabled) { cursors.delete(key); return []; }
if (await toCompany(scope, worker)) return { rows: null, complete: true };
const changedSince = passes.changedSince(scope, key, !cursors.has(key));
const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId,
issueIds: scope.issueIds } : {}, cursors.get(key) ?? null, PAGE_SIZE);
issueIds: scope.issueIds, ...(changedSince ? { changedSince } : {}) } : {}, cursors.get(key) ?? null, PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
passes.loaded(scope, complete);
passes.loaded(scope, key, complete);
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(),
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete };
},