Integrate bounded issue recovery and canonical scope hints

This commit is contained in:
Dotta committed 2026-10-30 19:00:47 -05:00
commit b29236d341
12 files changed
+289 -66

No files matched your search

@@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
/** Topics for existing durable queues; register intent before their writes. */
export const DELIVERY_QUEUES = {
orphanBlockers: "orphan-blockers",
activeRecoveryActions: "active-recovery-actions",
strandedAssignments: "stranded-assignments",
resolvedDependencies: "resolved-dependencies",
taskWatchdogs: "task-watchdogs",
+3 -1
View File
@@ -4245,7 +4245,9 @@ export function heartbeatService(
registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
registerIssueRecoveryWork(db, coordinator, {
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true }),
stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
recoveryActions: recovery.reconcileActiveRecoveryActions,
dependencies: reconcileResolvedDependencyWakes,
locks: sweepStaleIssueLocks,
watchdogs: reconcileTaskWatchdogs,
@@ -3,8 +3,8 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
export type IssueRecoveryScope = { companyId: string };
export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) {
const hint = scope ? JSON.stringify(scope) : "*";
for (const queue of [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) {
await notifyDeliveryWork(transaction, queue, hint);
}
@@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => {
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL");
if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-");
connectionString = external ?? temporary!.connectionString;
db = createDb(connectionString);
db = createDb(connectionString, { maxConnections: 6 });
await db.insert(companies).values([
{ id: companyId, name: "Native workspace owner", issuePrefix: "NWO" },
@@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => {
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" });
const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => {
await ownership.assertHeld();
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull();
expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull();
expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]);
return "copied";
});
@@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => {
try {
await entered;
await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`);
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run");
const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run");
expect(second).toEqual({ acquired: false, reason: "busy" });
const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy();
@@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => {
it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => {
const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" });
const interruptedDb = createDb(connectionString);
const interruptedDb = createDb(connectionString, { maxConnections: 6 });
let joined = false;
await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => {
await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work");
@@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => {
.toEqual({ acquired: false, reason: "owner_unverified" });
}
await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId));
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed"))
expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed"))
.toEqual({ acquired: true, value: "resumed" });
});
+20 -12
View File
@@ -2203,7 +2203,7 @@ export function recoveryService(
return Boolean(budgetBlock);
}
async function reconcileUnassignedBlockingIssues() {
async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) {
const candidates = await db
.select({
id: issues.id,
@@ -2217,6 +2217,8 @@ export function recoveryService(
.where(
and(
eq(issueRelations.type, "blocks"),
eq(issueRelations.companyId, issues.companyId),
opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
inArray(issues.status, ["todo", "blocked"]),
isNull(issues.assigneeAgentId),
isNull(issues.assigneeUserId),
@@ -3501,7 +3503,7 @@ export function recoveryService(
return Boolean(run || wake);
}
async function reconcileActiveRecoveryActions() {
async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) {
const rows = await db
.select({
action: issueRecoveryActions,
@@ -3515,7 +3517,8 @@ export function recoveryService(
eq(issues.companyId, issueRecoveryActions.companyId),
),
)
.where(inArray(issueRecoveryActions.status, ["active", "escalated"]));
.where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined));
const result = {
requeued: 0,
@@ -4532,6 +4535,7 @@ export function recoveryService(
onDeferredDeadline?: (at: number) => void;
issueCreatedAtGte?: Date | null;
skipExternalConversationSettlement?: boolean;
skipRelatedRecovery?: boolean;
}) {
const candidates = await db
.select(recoveryIssueFields)
@@ -5780,16 +5784,18 @@ export function recoveryService(
}
}
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
result.skipped += orphanBlockerRecovery.skipped;
result.issueIds.push(...orphanBlockerRecovery.issueIds);
if (!opts?.skipRelatedRecovery) {
const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts);
result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
result.skipped += orphanBlockerRecovery.skipped;
result.issueIds.push(...orphanBlockerRecovery.issueIds);
const activeRecovery = await reconcileActiveRecoveryActions();
result.continuationRequeued += activeRecovery.requeued;
result.escalated += activeRecovery.escalated;
result.skipped += activeRecovery.skipped;
result.issueIds.push(...activeRecovery.issueIds);
const activeRecovery = await reconcileActiveRecoveryActions(opts);
result.continuationRequeued += activeRecovery.requeued;
result.escalated += activeRecovery.escalated;
result.skipped += activeRecovery.skipped;
result.issueIds.push(...activeRecovery.issueIds);
}
result.issueIds = [...new Set(result.issueIds)];
return result;
@@ -6563,6 +6569,8 @@ export function recoveryService(
recordWatchdogDecision,
scanSilentActiveRuns,
reconcileStrandedAssignedIssues,
reconcileUnassignedBlockingIssues,
reconcileActiveRecoveryActions,
reconcileLegacyContinuation,
legacyRepairDispatchBlock,
sweepStaleIssueLocks,
+62 -8
View File
@@ -1,6 +1,6 @@
import { registerTaskWatchdogWork } from "../task-watchdog-work.js";
import { and, eq, inArray, isNotNull, isNull, or } from "drizzle-orm";
import { issues, type Db } from "@paperclipai/db";
import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
import type { IssueRecoveryScope } from "../issue-recovery-notifications.js";
@@ -8,12 +8,15 @@ import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
type Entry = { id: string; companyId: string; dueAt: number };
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void };
const PAGE_SIZE = 100;
export function registerIssueRecoveryWork(
db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
owner: {
stranded(scope: Scope): Promise<unknown>;
orphanBlockers(scope: { companyId: string }): Promise<unknown>;
recoveryActions(scope: { companyId: string }): Promise<unknown>;
dependencies(scope: Scope): Promise<{ enqueueFailed: number }>;
locks(scope: Scope): Promise<unknown>;
watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>;
@@ -22,20 +25,32 @@ export function registerIssueRecoveryWork(
canRun: () => boolean,
) {
function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise<unknown>) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
load: async scope => {
const key = scope?.companyId ?? "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
scope ? eq(issues.companyId, scope.companyId) : undefined,
cursor ? gt(issues.id, cursor) : undefined,
statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)),
statuses ? isNull(issues.hiddenAt) : undefined,
));
return rows.map(row => ({ ...row, dueAt: Date.now() }));
statuses ? isNull(issues.assigneeUserId) : undefined,
statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined,
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => {
if (await owner.schedulingSuppressed()) return null;
if (!enabled) return null;
let next: number | null = null;
await check({ companyId: entry.companyId, issueId: entry.id,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
@@ -43,7 +58,45 @@ export function registerIssueRecoveryWork(
},
}, canRun);
}
function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise<unknown>) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
load: async scope => {
const key = scope?.companyId ?? "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = kind === "actions"
? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId })
.from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined,
cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined))
.orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE)
: await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and(
inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId),
isNotNull(issues.createdByAgentId),
scope ? eq(issues.companyId, scope.companyId) : undefined,
cursor ? gt(issues.companyId, cursor) : undefined,
sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent
on dependent.id = r.related_issue_id and dependent.company_id = r.company_id
where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id}
and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`,
)).orderBy(asc(issues.companyId)).limit(PAGE_SIZE);
const complete = rows.length < PAGE_SIZE;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; },
}, canRun);
}
const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded);
const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers);
const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions);
const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => {
const result = await owner.dependencies(scope);
if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered");
@@ -52,7 +105,8 @@ export function registerIssueRecoveryWork(
const watchdogs = registerTaskWatchdogWork(db, coordinator, {
reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()),
}, canRun);
return { ready: Promise.all([assignments.ready, dependencies.ready, locks.ready, watchdogs.ready]),
wake() { assignments.invalidate(); dependencies.invalidate(); locks.invalidate(); watchdogs.invalidate(); },
async stop() { await Promise.all([assignments.stop(), dependencies.stop(), locks.stop(), watchdogs.stop()]); } };
const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs];
return { ready: Promise.all(workers.map(worker => worker.ready)),
wake() { for (const worker of workers) worker.invalidate(); },
async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
}
+18 -2
View File
@@ -12,6 +12,8 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
queue: DeliveryQueue;
batchSize?: number;
minimumPassIntervalMs?: number;
reconstructionIntervalMs?: number;
beforePass?(): Promise<void>;
decodeScope(hint: string): Scope | undefined;
matches(entry: Entry, scope?: Scope): boolean;
load(scope?: Scope, signal?: AbortSignal): Promise<Entry[] | { rows: Entry[] | null; complete: boolean }>;
@@ -25,6 +27,7 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
const entries = new Map<string, Entry>();
const failures = new Map<string, { count: number; retryAt: number }>();
const scopeFailures = new Map<string, { count: number; retryAt: number }>();
const reconstructAfter = new Map<string, number>();
let earliest = 0;
let lastCompany: string | null = null;
function failed(map: typeof failures, id: string) {
@@ -34,13 +37,14 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
function dirtyScopes() {
return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint))
.filter(hint => !scans.has(hint) || scopeFailures.has(hint))
.filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint))
.map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } });
}
function next() {
if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null;
let nextAt: number | null = null;
const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); };
for (const hint of dirty) take(Math.max(earliest, scopeFailures.get(hint)?.retryAt ?? 0));
for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0));
const unavailable = dirtyScopes();
for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope)))
take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0));
@@ -54,9 +58,18 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
} },
run: async signal => {
if (!canRun() || Date.now() < earliest) return;
if ((options.reconstructionIntervalMs ?? 0) > 0) for (const [hint, until] of reconstructAfter) {
if (until <= Date.now()) reconstructAfter.delete(hint);
}
const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return;
earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000);
try { await options.beforePass?.(); } catch (error) {
for (const hint of ready) failed(scopeFailures, hint);
for (const entry of entries.values()) if (entry.dueAt <= Date.now()) failed(failures, entry.id);
throw error;
}
const errors: unknown[] = [];
const ready = [...dirty].filter(hint => (scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now());
const requested = ready.includes("*")
? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25)
: ready.slice(0, 25);
@@ -81,6 +94,9 @@ export function registerScopedDeadlineWork<Entry extends ScheduledEntry, Scope>(
entries.delete(entry.id); failures.delete(entry.id);
}
scans.delete(hint);
if ((options.reconstructionIntervalMs ?? 0) > 0) {
reconstructAfter.set(hint, Date.now() + options.reconstructionIntervalMs!);
}
if ((versions.get(hint) ?? 0) !== scan.version) dirty.add(hint); else versions.delete(hint);
if (!scope) scopeFailures.clear(); else scopeFailures.delete(hint);
} catch (error) {
+19 -6
View File
@@ -1,4 +1,4 @@
import { and, eq } from "drizzle-orm";
import { and, asc, eq, gt } from "drizzle-orm";
import { issueWatchdogs, type Db } from "@paperclipai/db";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
@@ -16,16 +16,29 @@ export function registerTaskWatchdogWork(
},
canRun: () => boolean,
) {
const cursors = new Map<string, string>();
let enabled = false;
return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
queue: DELIVERY_QUEUES.taskWatchdogs,
reconstructionIntervalMs: 30_000,
beforePass: async () => { enabled = await owner.enabled(); },
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
load: async scope => (await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId })
.from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"),
scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined)))
.map(row => ({ ...row, dueAt: Date.now() })),
load: async scope => {
const key = scope?.companyId ?? "*";
if (!enabled) { cursors.delete(key); return []; }
const cursor = cursors.get(key);
const rows = await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId })
.from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"),
scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined,
cursor ? gt(issueWatchdogs.id, cursor) : undefined))
.orderBy(asc(issueWatchdogs.id)).limit(100);
const complete = rows.length < 100;
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
},
check: async entry => {
if (!(await owner.enabled())) return null;
if (!enabled) return null;
let next: number | null = null;
const result = await owner.reconcile({ companyId: entry.companyId, watchdogId: entry.id,
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });