mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 13:24:55 +01:00
Integrate bounded issue recovery and canonical scope hints
This commit is contained in:
commit
b29236d341
12 files changed
+289
-66
No files matched your search
@@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
|
||||
|
||||
/** Topics for existing durable queues; register intent before their writes. */
|
||||
export const DELIVERY_QUEUES = {
|
||||
orphanBlockers: "orphan-blockers",
|
||||
activeRecoveryActions: "active-recovery-actions",
|
||||
strandedAssignments: "stranded-assignments",
|
||||
resolvedDependencies: "resolved-dependencies",
|
||||
taskWatchdogs: "task-watchdogs",
|
||||
|
||||
@@ -4245,7 +4245,9 @@ export function heartbeatService(
|
||||
|
||||
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
|
||||
registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true }),
|
||||
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
|
||||
orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
|
||||
recoveryActions: recovery.reconcileActiveRecoveryActions,
|
||||
dependencies: reconcileResolvedDependencyWakes,
|
||||
locks: sweepStaleIssueLocks,
|
||||
watchdogs: reconcileTaskWatchdogs,
|
||||
|
||||
@@ -3,8 +3,8 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
|
||||
export type IssueRecoveryScope = { companyId: string };
|
||||
|
||||
export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) {
|
||||
const hint = scope ? JSON.stringify(scope) : "*";
|
||||
for (const queue of [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
|
||||
const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
|
||||
for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
|
||||
DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) {
|
||||
await notifyDeliveryWork(transaction, queue, hint);
|
||||
}
|
||||
|
||||
@@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-");
|
||||
connectionString = external ?? temporary!.connectionString;
|
||||
db = createDb(connectionString);
|
||||
db = createDb(connectionString, { maxConnections: 6 });
|
||||
|
||||
await db.insert(companies).values([
|
||||
{ id: companyId, name: "Native workspace owner", issuePrefix: "NWO" },
|
||||
@@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => {
|
||||
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" });
|
||||
const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => {
|
||||
await ownership.assertHeld();
|
||||
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull();
|
||||
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull();
|
||||
expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]);
|
||||
return "copied";
|
||||
});
|
||||
@@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => {
|
||||
try {
|
||||
await entered;
|
||||
await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`);
|
||||
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run");
|
||||
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run");
|
||||
expect(second).toEqual({ acquired: false, reason: "busy" });
|
||||
const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy();
|
||||
@@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => {
|
||||
|
||||
it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => {
|
||||
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" });
|
||||
const interruptedDb = createDb(connectionString);
|
||||
const interruptedDb = createDb(connectionString, { maxConnections: 6 });
|
||||
let joined = false;
|
||||
await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => {
|
||||
await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work");
|
||||
@@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => {
|
||||
.toEqual({ acquired: false, reason: "owner_unverified" });
|
||||
}
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId));
|
||||
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed"))
|
||||
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed"))
|
||||
.toEqual({ acquired: true, value: "resumed" });
|
||||
});
|
||||
|
||||
|
||||
@@ -2203,7 +2203,7 @@ export function recoveryService(
|
||||
return Boolean(budgetBlock);
|
||||
}
|
||||
|
||||
async function reconcileUnassignedBlockingIssues() {
|
||||
async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) {
|
||||
const candidates = await db
|
||||
.select({
|
||||
id: issues.id,
|
||||
@@ -2217,6 +2217,8 @@ export function recoveryService(
|
||||
.where(
|
||||
and(
|
||||
eq(issueRelations.type, "blocks"),
|
||||
eq(issueRelations.companyId, issues.companyId),
|
||||
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
|
||||
inArray(issues.status, ["todo", "blocked"]),
|
||||
isNull(issues.assigneeAgentId),
|
||||
isNull(issues.assigneeUserId),
|
||||
@@ -3501,7 +3503,7 @@ export function recoveryService(
|
||||
return Boolean(run || wake);
|
||||
}
|
||||
|
||||
async function reconcileActiveRecoveryActions() {
|
||||
async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) {
|
||||
const rows = await db
|
||||
.select({
|
||||
action: issueRecoveryActions,
|
||||
@@ -3515,7 +3517,8 @@ export function recoveryService(
|
||||
eq(issues.companyId, issueRecoveryActions.companyId),
|
||||
),
|
||||
)
|
||||
.where(inArray(issueRecoveryActions.status, ["active", "escalated"]));
|
||||
.where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined));
|
||||
|
||||
const result = {
|
||||
requeued: 0,
|
||||
@@ -4532,6 +4535,7 @@ export function recoveryService(
|
||||
onDeferredDeadline?: (at: number) => void;
|
||||
issueCreatedAtGte?: Date | null;
|
||||
skipExternalConversationSettlement?: boolean;
|
||||
skipRelatedRecovery?: boolean;
|
||||
}) {
|
||||
const candidates = await db
|
||||
.select(recoveryIssueFields)
|
||||
@@ -5780,16 +5784,18 @@ export function recoveryService(
|
||||
}
|
||||
}
|
||||
|
||||
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
|
||||
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
|
||||
result.skipped += orphanBlockerRecovery.skipped;
|
||||
result.issueIds.push(...orphanBlockerRecovery.issueIds);
|
||||
if (!opts?.skipRelatedRecovery) {
|
||||
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts);
|
||||
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
|
||||
result.skipped += orphanBlockerRecovery.skipped;
|
||||
result.issueIds.push(...orphanBlockerRecovery.issueIds);
|
||||
|
||||
const activeRecovery = await reconcileActiveRecoveryActions();
|
||||
result.continuationRequeued += activeRecovery.requeued;
|
||||
result.escalated += activeRecovery.escalated;
|
||||
result.skipped += activeRecovery.skipped;
|
||||
result.issueIds.push(...activeRecovery.issueIds);
|
||||
const activeRecovery = await reconcileActiveRecoveryActions(opts);
|
||||
result.continuationRequeued += activeRecovery.requeued;
|
||||
result.escalated += activeRecovery.escalated;
|
||||
result.skipped += activeRecovery.skipped;
|
||||
result.issueIds.push(...activeRecovery.issueIds);
|
||||
}
|
||||
result.issueIds = [...new Set(result.issueIds)];
|
||||
|
||||
return result;
|
||||
@@ -6563,6 +6569,8 @@ export function recoveryService(
|
||||
recordWatchdogDecision,
|
||||
scanSilentActiveRuns,
|
||||
reconcileStrandedAssignedIssues,
|
||||
reconcileUnassignedBlockingIssues,
|
||||
reconcileActiveRecoveryActions,
|
||||
reconcileLegacyContinuation,
|
||||
legacyRepairDispatchBlock,
|
||||
sweepStaleIssueLocks,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { registerTaskWatchdogWork } from "../task-watchdog-work.js";
|
||||
import { and, eq, inArray, isNotNull, isNull, or } from "drizzle-orm";
|
||||
import { issues, type Db } from "@paperclipai/db";
|
||||
import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
|
||||
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
|
||||
import type { IssueRecoveryScope } from "../issue-recovery-notifications.js";
|
||||
@@ -8,12 +8,15 @@ import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
|
||||
type Entry = { id: string; companyId: string; dueAt: number };
|
||||
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void };
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
export function registerIssueRecoveryWork(
|
||||
db: Db,
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
owner: {
|
||||
stranded(scope: Scope): Promise<unknown>;
|
||||
orphanBlockers(scope: { companyId: string }): Promise<unknown>;
|
||||
recoveryActions(scope: { companyId: string }): Promise<unknown>;
|
||||
dependencies(scope: Scope): Promise<{ enqueueFailed: number }>;
|
||||
locks(scope: Scope): Promise<unknown>;
|
||||
watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>;
|
||||
@@ -22,20 +25,32 @@ export function registerIssueRecoveryWork(
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise<unknown>) {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
|
||||
load: async scope => {
|
||||
const key = scope?.companyId ?? "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
|
||||
scope ? eq(issues.companyId, scope.companyId) : undefined,
|
||||
cursor ? gt(issues.id, cursor) : undefined,
|
||||
statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)),
|
||||
statuses ? isNull(issues.hiddenAt) : undefined,
|
||||
));
|
||||
return rows.map(row => ({ ...row, dueAt: Date.now() }));
|
||||
statuses ? isNull(issues.assigneeUserId) : undefined,
|
||||
statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined,
|
||||
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
|
||||
const complete = rows.length < PAGE_SIZE;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (await owner.schedulingSuppressed()) return null;
|
||||
if (!enabled) return null;
|
||||
let next: number | null = null;
|
||||
await check({ companyId: entry.companyId, issueId: entry.id,
|
||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
|
||||
@@ -43,7 +58,45 @@ export function registerIssueRecoveryWork(
|
||||
},
|
||||
}, canRun);
|
||||
}
|
||||
function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise<unknown>) {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
|
||||
queue,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
|
||||
load: async scope => {
|
||||
const key = scope?.companyId ?? "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = kind === "actions"
|
||||
? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId })
|
||||
.from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined,
|
||||
cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined))
|
||||
.orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE)
|
||||
: await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and(
|
||||
inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId),
|
||||
isNotNull(issues.createdByAgentId),
|
||||
scope ? eq(issues.companyId, scope.companyId) : undefined,
|
||||
cursor ? gt(issues.companyId, cursor) : undefined,
|
||||
sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent
|
||||
on dependent.id = r.related_issue_id and dependent.company_id = r.company_id
|
||||
where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id}
|
||||
and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`,
|
||||
)).orderBy(asc(issues.companyId)).limit(PAGE_SIZE);
|
||||
const complete = rows.length < PAGE_SIZE;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
|
||||
},
|
||||
check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; },
|
||||
}, canRun);
|
||||
}
|
||||
const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded);
|
||||
const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers);
|
||||
const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions);
|
||||
const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => {
|
||||
const result = await owner.dependencies(scope);
|
||||
if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered");
|
||||
@@ -52,7 +105,8 @@ export function registerIssueRecoveryWork(
|
||||
const watchdogs = registerTaskWatchdogWork(db, coordinator, {
|
||||
reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()),
|
||||
}, canRun);
|
||||
return { ready: Promise.all([assignments.ready, dependencies.ready, locks.ready, watchdogs.ready]),
|
||||
wake() { assignments.invalidate(); dependencies.invalidate(); locks.invalidate(); watchdogs.invalidate(); },
|
||||
async stop() { await Promise.all([assignments.stop(), dependencies.stop(), locks.stop(), watchdogs.stop()]); } };
|
||||
const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs];
|
||||
return { ready: Promise.all(workers.map(worker => worker.ready)),
|
||||
wake() { for (const worker of workers) worker.invalidate(); },
|
||||
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
|
||||
}
|
||||
@@ -12,6 +12,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
queue: DeliveryQueue;
|
||||
batchSize?: number;
|
||||
minimumPassIntervalMs?: number;
|
||||
reconstructionIntervalMs?: number;
|
||||
beforePass?(): Promise<void>;
|
||||
decodeScope(hint: string): Scope | undefined;
|
||||
matches(entry: Entry, scope?: Scope): boolean;
|
||||
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
|
||||
@@ -25,6 +27,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
const entries = new Map<string, Entry>();
|
||||
const failures = new Map<string, { count: number; retryAt: number }>();
|
||||
const scopeFailures = new Map<string, { count: number; retryAt: number }>();
|
||||
const reconstructAfter = new Map<string, number>();
|
||||
let earliest = 0;
|
||||
let lastCompany: string | null = null;
|
||||
function failed(map: typeof failures, id: string) {
|
||||
@@ -34,13 +37,14 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
function dirtyScopes() {
|
||||
return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint))
|
||||
.filter(hint => !scans.has(hint) || scopeFailures.has(hint))
|
||||
.filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint))
|
||||
.map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } });
|
||||
}
|
||||
function next() {
|
||||
if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null;
|
||||
let nextAt: number | null = null;
|
||||
const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); };
|
||||
for (const hint of dirty) take(Math.max(earliest, scopeFailures.get(hint)?.retryAt ?? 0));
|
||||
for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0));
|
||||
const unavailable = dirtyScopes();
|
||||
for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope)))
|
||||
take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0));
|
||||
@@ -54,9 +58,18 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
} },
|
||||
run: async signal => {
|
||||
if (!canRun() || Date.now() < earliest) return;
|
||||
if ((options.reconstructionIntervalMs ?? 0) > 0) for (const [hint, until] of reconstructAfter) {
|
||||
if (until <= Date.now()) reconstructAfter.delete(hint);
|
||||
}
|
||||
const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
|
||||
if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return;
|
||||
earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000);
|
||||
try { await options.beforePass?.(); } catch (error) {
|
||||
for (const hint of ready) failed(scopeFailures, hint);
|
||||
for (const entry of entries.values()) if (entry.dueAt <= Date.now()) failed(failures, entry.id);
|
||||
throw error;
|
||||
}
|
||||
const errors: unknown[] = [];
|
||||
const ready = [...dirty].filter(hint => (scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
|
||||
const requested = ready.includes("*")
|
||||
? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25)
|
||||
: ready.slice(0, 25);
|
||||
@@ -81,6 +94,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
|
||||
entries.delete(entry.id); failures.delete(entry.id);
|
||||
}
|
||||
scans.delete(hint);
|
||||
if ((options.reconstructionIntervalMs ?? 0) > 0) {
|
||||
reconstructAfter.set(hint, Date.now() + options.reconstructionIntervalMs!);
|
||||
}
|
||||
if ((versions.get(hint) ?? 0) !== scan.version) dirty.add(hint); else versions.delete(hint);
|
||||
if (!scope) scopeFailures.clear(); else scopeFailures.delete(hint);
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, asc, eq, gt } from "drizzle-orm";
|
||||
import { issueWatchdogs, type Db } from "@paperclipai/db";
|
||||
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
@@ -16,16 +16,29 @@ export function registerTaskWatchdogWork(
|
||||
},
|
||||
canRun: () => boolean,
|
||||
) {
|
||||
const cursors = new Map<string, string>();
|
||||
let enabled = false;
|
||||
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
|
||||
queue: DELIVERY_QUEUES.taskWatchdogs,
|
||||
reconstructionIntervalMs: 30_000,
|
||||
beforePass: async () => { enabled = await owner.enabled(); },
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
|
||||
load: async scope => (await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId })
|
||||
.from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"),
|
||||
scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined)))
|
||||
.map(row => ({ ...row, dueAt: Date.now() })),
|
||||
load: async scope => {
|
||||
const key = scope?.companyId ?? "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
const cursor = cursors.get(key);
|
||||
const rows = await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId })
|
||||
.from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"),
|
||||
scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined,
|
||||
cursor ? gt(issueWatchdogs.id, cursor) : undefined))
|
||||
.orderBy(asc(issueWatchdogs.id)).limit(100);
|
||||
const complete = rows.length < 100;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!(await owner.enabled())) return null;
|
||||
if (!enabled) return null;
|
||||
let next: number | null = null;
|
||||
const result = await owner.reconcile({ companyId: entry.companyId, watchdogId: entry.id,
|
||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
|
||||
|
||||
Reference in new issue
Block a user