Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-10-31 14:16:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): back off failed approval queue projections

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-24 23:55:39 -05:00
1 parent 0c7109c103
commit 0d3100665d
7 files changed
+159 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/services/approval-queue-seeding-work.ts
+50 -39
View File
@@ -1,6 +1,7 @@
import { and, asc, eq, gt, sql, type SQL } from "drizzle-orm";
import { approvals, companies, issues, type Db } from "@paperclipai/db";
import { approvals, companies, issues, issueWorkProducts, type Db } from "@paperclipai/db";
import { executionIssueCondition } from "./issue-visibility.js";
import { pullRequestSeedProductCondition } from "./decision-queue-seed-rules.js";
import { decisionQueueService } from "./decision-queues.js";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
@@ -10,12 +11,12 @@ const linkedIssue = sql<string>`(select linked.issue_id from issue_approvals lin
order by linked.issue_id limit 1)`;
export function approvalQueueSeedingService(db: Db) {
const pending = new Map<string | undefined, { cursor?: string; generation: number }>([[undefined, { generation: 0 }]]);
const pending = new Map<string | undefined, { cursor?: string; generation: number; failures: number; retryAt: number }>([[undefined, { generation: 0, failures: 0, retryAt: 0 }]]);
function request(companyIds: ReadonlySet<string> | undefined) {
for (const companyId of companyIds ?? []) {
const existing = pending.get(companyId);
if (existing) existing.generation++;
else pending.set(companyId, { generation: 0 });
else pending.set(companyId, { generation: 0, failures: 0, retryAt: 0 });
}
}
function candidates(database: Db, companyId: string | undefined, condition?: SQL) {
@@ -24,47 +25,56 @@ export function approvalQueueSeedingService(db: Db) {
.where(and(eq(approvals.status, "pending"), eq(companies.status, "active"), companyId === undefined ? undefined : eq(approvals.companyId, companyId), condition,
sql`exists (select 1 from ${issues} where ${issues.id} = ${linkedIssue}
and ${issues.companyId} = ${approvals.companyId} and ${executionIssueCondition()}
and exists (select 1 from issue_work_products product where product.company_id = ${approvals.companyId}
and product.issue_id = ${issues.id} and product.type = 'pull_request'))`,
and exists (select 1 from ${issueWorkProducts} where ${pullRequestSeedProductCondition(approvals.companyId)}
and ${issueWorkProducts.issueId} = ${issues.id}))`,
sql`not exists (select 1 from decision_queues queue where queue.company_id = ${approvals.companyId} and queue.key = 'prs'
and (not queue.seed_rules_enabled or exists (select 1 from decision_queue_items item where item.queue_id = queue.id
and item.company_id = ${approvals.companyId} and item.source_kind = 'approval' and item.source_id = ${approvals.id}::text)))`,
)).orderBy(asc(approvals.id));
}
async function sweep(signal?: AbortSignal) {
const scope = pending.entries().next().value;
const scope = [...pending.entries()].find(([, state]) => state.retryAt <= Date.now());
if (!scope) return;
const [companyId, state] = scope;
const generation = state.generation;
const rows = await candidates(db, companyId, state.cursor ? gt(approvals.id, state.cursor) : undefined).limit(100);
const errors: unknown[] = [];
for (const row of rows) {
if (signal?.aborted) break;
try {
await db.transaction(async tx => {
const database = tx as unknown as Db;
const [current] = await candidates(database, row.companyId, eq(approvals.id, row.id)).for("update", { of: approvals });
if (!current) return;
await decisionQueueService(database).materializeSeedSources(current.companyId, [{
sourceKind: "approval", sourceId: current.id, issueId: current.issueId,
}]);
});
} catch (error) { errors.push(error); }
state.cursor = row.id;
try {
const generation = state.generation;
const rows = await candidates(db, companyId, state.cursor ? gt(approvals.id, state.cursor) : undefined).limit(100);
const errors: unknown[] = [];
for (const row of rows) {
if (signal?.aborted) break;
try {
await db.transaction(async tx => {
const database = tx as unknown as Db;
const [current] = await candidates(database, row.companyId, eq(approvals.id, row.id)).for("update", { of: approvals });
if (!current) return;
await decisionQueueService(database).materializeSeedSources(current.companyId, [{
sourceKind: "approval", sourceId: current.id, issueId: current.issueId,
}]);
});
} catch (error) { errors.push(error); }
state.cursor = row.id;
}
if (rows.length < 100) state.cursor = undefined;
const remaining = await candidates(db, companyId).limit(1);
if (!signal?.aborted && generation === state.generation && remaining.length === 0) pending.delete(companyId);
if (errors.length === 1) throw errors[0];
if (errors.length) throw new AggregateError(errors, "Approval queue seeding failed");
state.retryAt = 0;
} catch (error) {
state.failures = Math.min(state.failures + 1, 7);
state.retryAt = Date.now() + Math.min(60_000, 1_000 * 2 ** (state.failures - 1));
throw error;
} finally {
if (pending.has(companyId)) {
pending.delete(companyId);
pending.set(companyId, state);
}
}
if (rows.length < 100) state.cursor = undefined;
const remaining = await candidates(db, companyId).limit(1);
if (!signal?.aborted && generation === state.generation && remaining.length === 0) {
pending.delete(companyId);
} else {
pending.delete(companyId);
pending.set(companyId, state);
}
if (errors.length === 1) throw errors[0];
if (errors.length) throw new AggregateError(errors, "Approval queue seeding failed");
}
async function next() {
return pending.size ? Date.now() : null;
function next() {
let earliest: number | null = null;
for (const state of pending.values()) earliest = earliest === null ? state.retryAt : Math.min(earliest, state.retryAt);
return earliest;
}
return { sweep, next, request };
}
@@ -75,19 +85,20 @@ export function registerApprovalQueueSeedingWork(
canRun: () => boolean,
) {
let earliest = 0;
let next: number | null = null;
return coordinator.register(DELIVERY_QUEUES.approvalQueueSeeding, {
retryMs: 1_000,
pausePassiveMaintenance: true,
onSettled: service.request,
run: async signal => {
if (!canRun()) return;
if (Date.now() < earliest) { next = earliest; return; }
if (Date.now() < earliest) return;
earliest = Date.now() + 1_000;
await service.sweep(signal);
next = await service.next();
},
hasPending: async () => (await service.next()) !== null,
nextRunAt: () => next,
hasPending: async () => (service.next() ?? Infinity) <= Date.now(),
nextRunAt: () => {
const next = service.next();
return next === null ? null : Math.max(earliest, next);
},
});
}
server/src/services/decision-queue-seed-rules.ts
+6
View File
@@ -0,0 +1,6 @@
import { and, eq, type SQLWrapper } from "drizzle-orm";
import { issueWorkProducts } from "@paperclipai/db";
export function pullRequestSeedProductCondition(companyId: string | SQLWrapper) {
return and(eq(issueWorkProducts.companyId, companyId), eq(issueWorkProducts.type, "pull_request"));
}
server/src/services/decision-queues.ts
+2 -2
View File
@@ -1,3 +1,4 @@
import { pullRequestSeedProductCondition } from "./decision-queue-seed-rules.js";
import { notifyApprovalQueueWork } from "./approval-work-notifications.js";
import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -402,8 +403,7 @@ export function decisionQueueService(db: Db) {
.select({ issueId: issueWorkProducts.issueId })
.from(issueWorkProducts)
.where(and(
eq(issueWorkProducts.companyId, companyId),
eq(issueWorkProducts.type, "pull_request"),
pullRequestSeedProductCondition(companyId),
inArray(issueWorkProducts.issueId, issueIds),
)).then((rows) => rows.map((row) => row.issueId)));