mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
perf(recovery): re-check only changed tasks between hourly full backstop passes
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6400698a8e
commit
379dda9e26
8 files changed
+284
-19
No files matched your search
@@ -0,0 +1,54 @@
|
|||||||
|
import { agents, heartbeatRuns, issueRecoveryActions, issues, type Db } from "@paperclipai/db";
|
||||||
|
|
||||||
|
const HOUR = 60 * 60_000;
|
||||||
|
|
||||||
|
export type RecoveryPopulation = {
|
||||||
|
agentIds: string[];
|
||||||
|
pausedAgentId: string;
|
||||||
|
terminatedAgentId: string;
|
||||||
|
inReview: string[];
|
||||||
|
inProgress: string[];
|
||||||
|
todo: string[];
|
||||||
|
withRecoveryAction: string[];
|
||||||
|
terminated: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function seedRecoveryPopulation(db: Db, companyId: string, counts: {
|
||||||
|
agents: number; inReview: number; inProgress?: number; todo?: number; recoveryActions?: number; terminated?: number;
|
||||||
|
}): Promise<RecoveryPopulation> {
|
||||||
|
const config = { adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() },
|
||||||
|
runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } }, role: "engineer" };
|
||||||
|
const [ceo] = await db.insert(agents).values({ ...config, companyId, name: "Chief", status: "idle" }).returning({ id: agents.id });
|
||||||
|
const managers = await db.insert(agents).values(Array.from({ length: 6 }, (_, index) => ({ ...config, companyId,
|
||||||
|
name: `Manager ${index}`, status: "idle", reportsTo: ceo!.id }))).returning({ id: agents.id });
|
||||||
|
const workers = await db.insert(agents).values(Array.from({ length: Math.max(1, counts.agents - 9) }, (_, index) => ({ ...config,
|
||||||
|
companyId, name: `Worker ${index}`, status: "idle", reportsTo: managers[index % managers.length]!.id }))).returning({ id: agents.id });
|
||||||
|
const [paused] = await db.insert(agents).values({ ...config, companyId, name: "Paused worker", status: "paused",
|
||||||
|
lifecycleHolds: ["manual"], reportsTo: managers[0]!.id }).returning({ id: agents.id });
|
||||||
|
const [terminatedAgent] = await db.insert(agents).values({ ...config, companyId, name: "Former worker", status: "terminated",
|
||||||
|
reportsTo: managers[1]!.id }).returning({ id: agents.id });
|
||||||
|
const workerIds = workers.map(row => row.id);
|
||||||
|
const finishedAt = new Date(Date.now() - HOUR);
|
||||||
|
|
||||||
|
const insertTasks = async (count: number, values: (index: number) => Partial<typeof issues.$inferInsert>) => count === 0 ? []
|
||||||
|
: (await db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ companyId, title: `Task ${index}`,
|
||||||
|
...values(index) }))).returning({ id: issues.id, assigneeAgentId: issues.assigneeAgentId })).map(row => row);
|
||||||
|
|
||||||
|
const reviews = await insertTasks(counts.inReview, index => ({ status: "in_review",
|
||||||
|
assigneeAgentId: workerIds[index % workerIds.length] }));
|
||||||
|
if (reviews.length) await db.insert(heartbeatRuns).values(reviews.map(task => ({ companyId, agentId: task.assigneeAgentId!,
|
||||||
|
status: "succeeded", runtimeMode: "native", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } })));
|
||||||
|
const progressing = await insertTasks(counts.inProgress ?? 0, () => ({ status: "in_progress", assigneeAgentId: paused!.id }));
|
||||||
|
if (progressing.length) await db.insert(heartbeatRuns).values(progressing.map(task => ({ companyId, agentId: paused!.id,
|
||||||
|
status: "cancelled", errorCode: "agent_paused", startedAt: finishedAt, finishedAt, contextSnapshot: { issueId: task.id } })));
|
||||||
|
const todo = await insertTasks(counts.todo ?? 0, () => ({ status: "todo", assigneeAgentId: paused!.id }));
|
||||||
|
const terminated = await insertTasks(counts.terminated ?? 0, () => ({ status: "todo", assigneeAgentId: terminatedAgent!.id }));
|
||||||
|
const withAction = reviews.slice(0, counts.recoveryActions ?? 0);
|
||||||
|
if (withAction.length) await db.insert(issueRecoveryActions).values(withAction.map((task, index) => ({ companyId,
|
||||||
|
sourceIssueId: task.id, kind: "missing_disposition", status: "active", ownerType: "agent", ownerAgentId: task.assigneeAgentId,
|
||||||
|
cause: "seeded", fingerprint: `seeded-${index}`, nextAction: "seeded" })));
|
||||||
|
return { agentIds: [ceo!.id, ...managers.map(row => row.id), ...workerIds, paused!.id, terminatedAgent!.id],
|
||||||
|
pausedAgentId: paused!.id, terminatedAgentId: terminatedAgent!.id,
|
||||||
|
inReview: reviews.map(row => row.id), inProgress: progressing.map(row => row.id), todo: todo.map(row => row.id),
|
||||||
|
withRecoveryAction: withAction.map(row => row.id), terminated: terminated.map(row => row.id) };
|
||||||
|
}
|
||||||
@@ -113,7 +113,7 @@ describeIssueRecovery("issue recovery production producers and deadlines", () =>
|
|||||||
const [run] = await state.db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", finishedAt: new Date(),
|
const [run] = await state.db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", finishedAt: new Date(),
|
||||||
contextSnapshot: { issueId: task.id } }).returning({ id: heartbeatRuns.id });
|
contextSnapshot: { issueId: task.id } }).returning({ id: heartbeatRuns.id });
|
||||||
const backlog = async (count: number) => (await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({
|
const backlog = async (count: number) => (await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({
|
||||||
companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id,
|
companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id, createdAt: new Date(), updatedAt: new Date(),
|
||||||
}))).returning({ id: issues.id })).map(row => row.id);
|
}))).returning({ id: issues.id })).map(row => row.id);
|
||||||
return { runId: run.id, taskId: task.id, siblingId: sibling.id, backlog, affected: [parent.id, task.id, dependent.id].sort() };
|
return { runId: run.id, taskId: task.id, siblingId: sibling.id, backlog, affected: [parent.id, task.id, dependent.id].sort() };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,166 @@
|
|||||||
|
import { setImmediate as immediate } from "node:timers/promises";
|
||||||
|
import { and, eq, sql } from "drizzle-orm";
|
||||||
|
import { expect, it, vi } from "vitest";
|
||||||
|
import { agents, heartbeatRuns, issues } from "@paperclipai/db";
|
||||||
|
import { workSignalDebug } from "@paperclipai/db/work-signal-debug";
|
||||||
|
import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, notifyIssueGraphRecovery }
|
||||||
|
from "../services/issue-recovery-notifications.js";
|
||||||
|
import { issueService } from "../services/issues.js";
|
||||||
|
import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js";
|
||||||
|
import { promoteDeferredWorkIntents } from "../services/work-intent-outbox.js";
|
||||||
|
import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js";
|
||||||
|
import { seedRecoveryPopulation } from "./helpers/recovery-population.js";
|
||||||
|
|
||||||
|
vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1"; });
|
||||||
|
|
||||||
|
const MINUTE = 60_000;
|
||||||
|
type Recorded = { at: number; name: string; fields: Record<string, unknown> };
|
||||||
|
|
||||||
|
describeIssueRecovery("stranded-task checks at idle for a staging-shaped company", () => {
|
||||||
|
const state = issueRecoveryFixture({ meter: true });
|
||||||
|
|
||||||
|
async function advance(ms: number) {
|
||||||
|
const until = Date.now() + ms;
|
||||||
|
while (Date.now() < until) {
|
||||||
|
await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now()));
|
||||||
|
await Promise.allSettled([...state.passes]); await immediate();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function record() {
|
||||||
|
const events: Recorded[] = [];
|
||||||
|
vi.spyOn(workSignalDebug, "event").mockImplementation((topic, name, fields = {}) => {
|
||||||
|
if (topic === "stranded-assignments") events.push({ at: Date.now(), name, fields });
|
||||||
|
});
|
||||||
|
vi.spyOn(workSignalDebug, "signal").mockImplementation(() => {});
|
||||||
|
return events;
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarize(events: Recorded[]) {
|
||||||
|
const checks: Record<string, number> = {}, loads: Record<string, number> = {};
|
||||||
|
for (const event of events) {
|
||||||
|
if (event.name === "check") checks[String(event.fields.wake)] = (checks[String(event.fields.wake)] ?? 0) + 1;
|
||||||
|
if (event.name === "load") {
|
||||||
|
const scope = String(event.fields.scope);
|
||||||
|
const kind = scope === "*" ? "reload" : scope.endsWith(",true]") ? "backstop" : /^\["[^"]+",null,null,null,null\]$/.test(scope)
|
||||||
|
? "company" : "hint";
|
||||||
|
loads[kind] = (loads[kind] ?? 0) + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { checks, loads, totalChecks: Object.values(checks).reduce((sum, count) => sum + count, 0) };
|
||||||
|
}
|
||||||
|
|
||||||
|
const checkedAt = (events: Recorded[], issueId: string) =>
|
||||||
|
events.find(event => event.name === "check" && event.fields.issueId === issueId)?.at ?? null;
|
||||||
|
|
||||||
|
async function seed() {
|
||||||
|
const f = await state.fixture();
|
||||||
|
const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 50, inReview: 780, inProgress: 40, todo: 30,
|
||||||
|
recoveryActions: 12, terminated: 3 });
|
||||||
|
const [side] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Finished side task", status: "done" })
|
||||||
|
.returning({ id: issues.id });
|
||||||
|
const activity = () => state.db.transaction(tx => notifyIssueGraphRecovery(tx, { companyId: f.companyId, issueId: side!.id }));
|
||||||
|
return { ...f, population, activity };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function boot() {
|
||||||
|
const f = await seed();
|
||||||
|
const events = record();
|
||||||
|
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||||
|
await state.start({});
|
||||||
|
await advance(10 * MINUTE);
|
||||||
|
return { ...f, events, boot: summarize(events.splice(0)) };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function heldTask(f: Awaited<ReturnType<typeof boot>>) {
|
||||||
|
const old = new Date(Date.now() - 2 * 60 * MINUTE);
|
||||||
|
const [held] = await state.db.insert(agents).values({ companyId: f.companyId, name: "Held worker", role: "engineer", status: "paused",
|
||||||
|
lifecycleHolds: ["manual"], adapterType: "process", reportsTo: f.population.agentIds[1], createdAt: old, updatedAt: old,
|
||||||
|
adapterConfig: { command: process.execPath, args: ["-e", "0"], cwd: process.cwd() },
|
||||||
|
runtimeConfig: { heartbeat: { enabled: false, intervalSec: 0, wakeOnDemand: true } } }).returning({ id: agents.id });
|
||||||
|
const [waiting] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Waits for its agent", status: "todo",
|
||||||
|
assigneeAgentId: held!.id, createdAt: old, updatedAt: old }).returning({ id: issues.id });
|
||||||
|
return { held: held!, waiting: waiting! };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function trickle(f: Awaited<ReturnType<typeof boot>>, minutes: number) {
|
||||||
|
const before = state.meter.read();
|
||||||
|
for (let minute = 0; minute < minutes; minute++) { await f.activity(); await advance(MINUTE); }
|
||||||
|
const after = state.meter.read();
|
||||||
|
const raw = f.events.splice(0);
|
||||||
|
return { ...summarize(raw), raw, statements: after.statements - before.statements,
|
||||||
|
bytes: after.bytesReceived - before.bytesReceived };
|
||||||
|
}
|
||||||
|
|
||||||
|
it("checks only changed tasks between hourly full passes while unrelated activity continues", async () => {
|
||||||
|
const f = await boot();
|
||||||
|
const between = await trickle(f, 30);
|
||||||
|
const hour = await trickle(f, 30);
|
||||||
|
const { raw: _first, ...firstHalfHour } = between, { raw: _second, ...secondHalfHour } = hour;
|
||||||
|
console.info(JSON.stringify({ boot: f.boot, firstHalfHour, secondHalfHour,
|
||||||
|
perSecond: { checks: (between.totalChecks + hour.totalChecks) / 3600, statements: (between.statements + hour.statements) / 3600 } }));
|
||||||
|
expect(f.boot.totalChecks).toBeGreaterThanOrEqual(f.population.inReview.length);
|
||||||
|
expect(between.totalChecks).toBe(0);
|
||||||
|
expect(between.statements).toBeLessThan(2_000);
|
||||||
|
expect(hour.checks.backstop ?? 0).toBeLessThanOrEqual(f.population.inReview.length + 100);
|
||||||
|
}, 1_800_000);
|
||||||
|
|
||||||
|
it("recovers a task whose hint was never sent within the backstop bound", async () => {
|
||||||
|
const f = await boot();
|
||||||
|
const worker = f.population.agentIds[7]!;
|
||||||
|
const now = new Date();
|
||||||
|
const [missed] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Created without a hint", status: "todo",
|
||||||
|
assigneeAgentId: worker, createdAt: now, updatedAt: now }).returning({ id: issues.id });
|
||||||
|
const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_BACKSTOP_MS + MINUTE) / MINUTE) + 1);
|
||||||
|
const checked = checkedAt(window.raw, missed!.id);
|
||||||
|
expect(checked).not.toBeNull();
|
||||||
|
expect(checked! - now.getTime()).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 2 * MINUTE);
|
||||||
|
const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||||
|
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${missed!.id}`));
|
||||||
|
expect(runs.length).toBeGreaterThan(0);
|
||||||
|
}, 1_800_000);
|
||||||
|
|
||||||
|
it("re-checks a task whose only change is outside its own rows within the hourly full pass", async () => {
|
||||||
|
const f = await boot();
|
||||||
|
const { held, waiting } = await heldTask(f);
|
||||||
|
await state.db.execute(sql`update agents set status = 'idle', lifecycle_holds = '[]'::jsonb where id = ${held.id}`);
|
||||||
|
const changedAt = Date.now();
|
||||||
|
const window = await trickle(f, Math.ceil((ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE) / MINUTE));
|
||||||
|
const checks = window.raw.filter(event => event.name === "check" && event.fields.issueId === waiting.id);
|
||||||
|
expect(checks.length).toBeGreaterThan(0);
|
||||||
|
expect(checks[0]!.at - changedAt).toBeGreaterThan(2 * ISSUE_RECOVERY_BACKSTOP_MS);
|
||||||
|
expect(checks[0]!.at - changedAt).toBeLessThanOrEqual(ISSUE_RECOVERY_FULL_BACKSTOP_MS + 2 * MINUTE);
|
||||||
|
const runs = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||||
|
.where(and(eq(heartbeatRuns.companyId, f.companyId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${waiting.id}`));
|
||||||
|
expect(runs.length).toBeGreaterThan(0);
|
||||||
|
}, 1_800_000);
|
||||||
|
|
||||||
|
it("picks up a finished run, a user message, a raised budget and a resumed agent within a minute", async () => {
|
||||||
|
const f = await boot();
|
||||||
|
await trickle(f, 6);
|
||||||
|
const [target, messaged] = f.population.inReview;
|
||||||
|
const within = async (issueId: string, act: () => Promise<unknown>) => {
|
||||||
|
f.events.length = 0;
|
||||||
|
const at = Date.now();
|
||||||
|
await act();
|
||||||
|
await advance(60_000);
|
||||||
|
const checked = checkedAt(f.events, issueId);
|
||||||
|
expect(checked, issueId).not.toBeNull();
|
||||||
|
expect(checked! - at).toBeLessThanOrEqual(60_000);
|
||||||
|
};
|
||||||
|
const [run] = await state.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||||
|
.where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${target!}`).limit(1);
|
||||||
|
await within(target!, () => state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: run!.id })));
|
||||||
|
await advance(31_000);
|
||||||
|
await within(messaged!, () => issueService(state.db).addComment(messaged!, "Please look again", { userId: "recovery-user" }));
|
||||||
|
await advance(31_000);
|
||||||
|
await within(target!, () => state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" })));
|
||||||
|
await advance(31_000);
|
||||||
|
const { held, waiting } = await heldTask(f);
|
||||||
|
await advance(31_000);
|
||||||
|
await within(waiting.id, async () => {
|
||||||
|
await state.db.update(agents).set({ status: "idle", lifecycleHolds: [], updatedAt: new Date() }).where(eq(agents.id, held.id));
|
||||||
|
await state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "pause_lifted" }));
|
||||||
|
});
|
||||||
|
}, 1_800_000);
|
||||||
|
});
|
||||||
@@ -11,6 +11,7 @@ import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js
|
|||||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||||
import { issueService } from "../services/issues.js";
|
import { issueService } from "../services/issues.js";
|
||||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||||
|
import { meterDatabaseStatements } from "./helpers/database-statement-meter.js";
|
||||||
|
|
||||||
vi.setConfig({ testTimeout: 90_000 });
|
vi.setConfig({ testTimeout: 90_000 });
|
||||||
|
|
||||||
@@ -18,7 +19,8 @@ const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_D
|
|||||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||||
export const describeIssueRecovery: ReturnType<typeof describe.skipIf> = describe.skipIf(!support.supported);
|
export const describeIssueRecovery: ReturnType<typeof describe.skipIf> = describe.skipIf(!support.supported);
|
||||||
|
|
||||||
export function issueRecoveryFixture() {
|
export function issueRecoveryFixture(options: { meter?: boolean } = {}) {
|
||||||
|
let meter: ReturnType<typeof meterDatabaseStatements> | undefined;
|
||||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||||
let db: ReturnType<typeof createDb>;
|
let db: ReturnType<typeof createDb>;
|
||||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||||
@@ -35,6 +37,7 @@ export function issueRecoveryFixture() {
|
|||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
|
if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
|
||||||
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||||
|
if (options.meter) meter = meterDatabaseStatements(db, external ?? database!.connectionString);
|
||||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||||
}, 90_000);
|
}, 90_000);
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
@@ -49,7 +52,7 @@ export function issueRecoveryFixture() {
|
|||||||
vi.useRealTimers(); vi.restoreAllMocks();
|
vi.useRealTimers(); vi.restoreAllMocks();
|
||||||
expect(errors).toEqual([]);
|
expect(errors).toEqual([]);
|
||||||
});
|
});
|
||||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
afterAll(async () => { meter?.uninstall(); await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||||
async function start(options: { backstopMs?: number } = { backstopMs: 0 }) {
|
async function start(options: { backstopMs?: number } = { backstopMs: 0 }) {
|
||||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||||
onError: error => errors.push(error) });
|
onError: error => errors.push(error) });
|
||||||
@@ -232,6 +235,7 @@ export function issueRecoveryFixture() {
|
|||||||
return {
|
return {
|
||||||
start, compete, fixture, completingProvider, settled, quiet, finish, completed, savedConversation,
|
start, compete, fixture, completingProvider, settled, quiet, finish, completed, savedConversation,
|
||||||
get db() { return db; },
|
get db() { return db; },
|
||||||
|
get meter() { return meter!; },
|
||||||
get heartbeat() { return heartbeat; },
|
get heartbeat() { return heartbeat; },
|
||||||
set heartbeat(value: typeof heartbeat) { heartbeat = value; },
|
set heartbeat(value: typeof heartbeat) { heartbeat = value; },
|
||||||
get coordinator() { return coordinator; },
|
get coordinator() { return coordinator; },
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ export type IssueRecoveryScope = { companyId: string; issueId?: string; graphRun
|
|||||||
|
|
||||||
export const ISSUE_RECOVERY_MAX_ISSUE_IDS = 100;
|
export const ISSUE_RECOVERY_MAX_ISSUE_IDS = 100;
|
||||||
export const ISSUE_RECOVERY_BACKSTOP_MS = 5 * 60_000;
|
export const ISSUE_RECOVERY_BACKSTOP_MS = 5 * 60_000;
|
||||||
|
export const ISSUE_RECOVERY_FULL_BACKSTOP_MS = 60 * 60_000;
|
||||||
const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$";
|
const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$";
|
||||||
|
|
||||||
const COMPANY_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.taskWatchdogs];
|
const COMPANY_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.taskWatchdogs];
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import {
|
|||||||
import { logger } from "../../middleware/logger.js";
|
import { logger } from "../../middleware/logger.js";
|
||||||
import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
|
import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js";
|
||||||
import { evaluateAgentInvokability } from "../agent-invokability.js";
|
import { evaluateAgentInvokability } from "../agent-invokability.js";
|
||||||
|
import { changedTaskCondition } from "./changed-tasks.js";
|
||||||
import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js";
|
import { isAgentAwaitingSetup } from "../../modules/agent-lifecycle/index.js";
|
||||||
import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js";
|
import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js";
|
||||||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||||
@@ -510,7 +511,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
|||||||
order by lr.created_at desc limit 1), false)`;
|
order by lr.created_at desc limit 1), false)`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null },
|
async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null;
|
||||||
|
changedSince?: Date },
|
||||||
cursor: string | null, limit: number) {
|
cursor: string | null, limit: number) {
|
||||||
const at = now();
|
const at = now();
|
||||||
const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId}
|
const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId}
|
||||||
@@ -530,6 +532,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) {
|
|||||||
scope.issueIds ? inArray(issues.id, scope.issueIds) : undefined,
|
scope.issueIds ? inArray(issues.id, scope.issueIds) : undefined,
|
||||||
cursor ? sql`${issues.id} > ${cursor}` : undefined,
|
cursor ? sql`${issues.id} > ${cursor}` : undefined,
|
||||||
scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined,
|
scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined,
|
||||||
|
scope.changedSince ? changedTaskCondition(scope.changedSince) : undefined,
|
||||||
inArray(issues.status, [...OPEN_STATUSES]),
|
inArray(issues.status, [...OPEN_STATUSES]),
|
||||||
sql`${issues.assigneeAgentId} is not null`,
|
sql`${issues.assigneeAgentId} is not null`,
|
||||||
isNull(issues.assigneeUserId),
|
isNull(issues.assigneeUserId),
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { sql } from "drizzle-orm";
|
||||||
|
import { issues } from "@paperclipai/db";
|
||||||
|
|
||||||
|
export function changedTaskCondition(since: Date) {
|
||||||
|
const at = sql`${since.toISOString()}::timestamptz`;
|
||||||
|
return sql`(${issues.updatedAt} >= ${at}
|
||||||
|
or exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId}
|
||||||
|
and (r.context_snapshot->>'issueId') = ${issues.id}::text and r.updated_at >= ${at})
|
||||||
|
or exists (select 1 from issue_recovery_actions a where a.company_id = ${issues.companyId}
|
||||||
|
and a.source_issue_id = ${issues.id} and a.updated_at >= ${at})
|
||||||
|
or exists (select 1 from agent_wakeup_requests w where w.company_id = ${issues.companyId}
|
||||||
|
and (w.payload->>'issueId') = ${issues.id}::text and w.updated_at >= ${at})
|
||||||
|
or exists (select 1 from issue_thread_interactions t where t.company_id = ${issues.companyId}
|
||||||
|
and t.issue_id = ${issues.id} and t.updated_at >= ${at})
|
||||||
|
or exists (select 1 from issue_comments c where c.company_id = ${issues.companyId}
|
||||||
|
and c.issue_id = ${issues.id} and c.created_at >= ${at})
|
||||||
|
or exists (select 1 from agents ag where ag.id = ${issues.assigneeAgentId} and ag.updated_at >= ${at}))`;
|
||||||
|
}
|
||||||
@@ -3,13 +3,14 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from
|
|||||||
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
|
import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
|
||||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||||
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
|
import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
|
||||||
import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope }
|
import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_FULL_BACKSTOP_MS, issueRecoveryBackstopHint, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope }
|
||||||
from "../issue-recovery-notifications.js";
|
from "../issue-recovery-notifications.js";
|
||||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||||
import { workSignalDebug } from "@paperclipai/db/work-signal-debug";
|
import { workSignalDebug } from "@paperclipai/db/work-signal-debug";
|
||||||
import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js";
|
import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js";
|
||||||
|
import { changedTaskCondition } from "./changed-tasks.js";
|
||||||
|
|
||||||
type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true };
|
type Entry = { id: string; companyId: string; dueAt: number; fromDeadline?: true; origin?: "reload" | "backstop" | "company" | "hint" };
|
||||||
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void };
|
type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number, detail?: { unchanged: boolean }) => void };
|
||||||
const PAGE_SIZE = 100;
|
const PAGE_SIZE = 100;
|
||||||
|
|
||||||
@@ -33,22 +34,35 @@ export function registerIssueRecoveryWork(
|
|||||||
schedulingSuppressed(): Promise<boolean>;
|
schedulingSuppressed(): Promise<boolean>;
|
||||||
/** The assigned-work invariant (assigned-work-sweep.ts). */
|
/** The assigned-work invariant (assigned-work-sweep.ts). */
|
||||||
assignedWork?: {
|
assignedWork?: {
|
||||||
listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[] }, cursor: string | null, limit: number):
|
listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; changedSince?: Date }, cursor: string | null, limit: number):
|
||||||
Promise<Array<{ id: string; companyId: string; updatedAt: Date }>>;
|
Promise<Array<{ id: string; companyId: string; updatedAt: Date }>>;
|
||||||
checkIssue(scope: Scope): Promise<unknown>;
|
checkIssue(scope: Scope): Promise<unknown>;
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
canRun: () => boolean,
|
canRun: () => boolean,
|
||||||
options: { backstopMs?: number } = {},
|
options: { backstopMs?: number; fullBackstopMs?: number } = {},
|
||||||
) {
|
) {
|
||||||
|
const backstopMs = options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS;
|
||||||
|
const fullBackstopMs = options.fullBackstopMs ?? options.backstopMs ?? ISSUE_RECOVERY_FULL_BACKSTOP_MS;
|
||||||
function fullPasses() {
|
function fullPasses() {
|
||||||
const completedAt = new Map<string, number>();
|
const completedAt = new Map<string, number>(), changedAt = new Map<string, number>();
|
||||||
|
const scans = new Map<string, Date | null>();
|
||||||
|
const lastFull = (companyId: string) => Math.max(completedAt.get("*") ?? 0, completedAt.get(companyId) ?? 0);
|
||||||
|
const lastLook = (companyId: string) => Math.max(lastFull(companyId), changedAt.get(companyId) ?? 0);
|
||||||
return {
|
return {
|
||||||
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop
|
backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop ? lastLook(scope.companyId) + backstopMs : null,
|
||||||
? Math.max(completedAt.get("*") ?? 0, completedAt.get(scope.companyId) ?? 0) + (options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS)
|
changedSince(scope: IssueRecoveryScope | undefined, key: string, firstPage: boolean): Date | null {
|
||||||
: null,
|
if (!scope?.backstop) return null;
|
||||||
loaded(scope: IssueRecoveryScope | undefined, complete: boolean) {
|
if (firstPage || !scans.has(key)) scans.set(key, Date.now() - lastFull(scope.companyId) >= fullBackstopMs ? null
|
||||||
if (complete && !scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now());
|
: new Date(lastLook(scope.companyId) - ISSUE_RECOVERY_BACKSTOP_MS));
|
||||||
|
return scans.get(key) ?? null;
|
||||||
|
},
|
||||||
|
loaded(scope: IssueRecoveryScope | undefined, key: string, complete: boolean) {
|
||||||
|
if (!complete) return;
|
||||||
|
const changed = scans.get(key) ?? null;
|
||||||
|
scans.delete(key);
|
||||||
|
if (changed && scope) changedAt.set(scope.companyId, Date.now());
|
||||||
|
else if (!scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now());
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -94,8 +108,10 @@ export function registerIssueRecoveryWork(
|
|||||||
if (!enabled) { cursors.delete(key); return []; }
|
if (!enabled) { cursors.delete(key); return []; }
|
||||||
if (await toCompany(scope, worker)) return { rows: null, complete: true };
|
if (await toCompany(scope, worker)) return { rows: null, complete: true };
|
||||||
const cursor = cursors.get(key);
|
const cursor = cursors.get(key);
|
||||||
|
const changedSince = passes.changedSince(scope, key, !cursor);
|
||||||
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
|
const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
|
||||||
scope ? eq(issues.companyId, scope.companyId) : undefined,
|
scope ? eq(issues.companyId, scope.companyId) : undefined,
|
||||||
|
changedSince ? changedTaskCondition(changedSince) : undefined,
|
||||||
scope?.issueId ? eq(issues.id, scope.issueId) : undefined,
|
scope?.issueId ? eq(issues.id, scope.issueId) : undefined,
|
||||||
scope?.issueIds ? inArray(issues.id, scope.issueIds) : undefined,
|
scope?.issueIds ? inArray(issues.id, scope.issueIds) : undefined,
|
||||||
cursor ? gt(issues.id, cursor) : undefined,
|
cursor ? gt(issues.id, cursor) : undefined,
|
||||||
@@ -107,13 +123,15 @@ export function registerIssueRecoveryWork(
|
|||||||
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
|
)).orderBy(asc(issues.id)).limit(PAGE_SIZE);
|
||||||
const complete = rows.length < PAGE_SIZE;
|
const complete = rows.length < PAGE_SIZE;
|
||||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||||
passes.loaded(scope, complete);
|
passes.loaded(scope, key, complete);
|
||||||
return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
|
const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const
|
||||||
|
: scope.issueId || isGraph(scope) ? "hint" as const : "company" as const;
|
||||||
|
return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete };
|
||||||
},
|
},
|
||||||
check: async entry => {
|
check: async entry => {
|
||||||
if (!enabled) return null;
|
if (!enabled) return null;
|
||||||
if (workSignalDebug.enabled) workSignalDebug.event(queue, "check", { issueId: entry.id, companyId: entry.companyId,
|
if (workSignalDebug.enabled) workSignalDebug.event(queue, "check", { issueId: entry.id, companyId: entry.companyId,
|
||||||
wake: entry.fromDeadline ? "deadline" : "load", dueAt: entry.dueAt });
|
wake: entry.fromDeadline ? "deadline" : entry.origin ?? "load", dueAt: entry.dueAt });
|
||||||
let next: number | null = null;
|
let next: number | null = null;
|
||||||
await check({ companyId: entry.companyId, issueId: entry.id,
|
await check({ companyId: entry.companyId, issueId: entry.id,
|
||||||
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
|
onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
|
||||||
@@ -145,11 +163,12 @@ export function registerIssueRecoveryWork(
|
|||||||
const key = scopeKey(scope);
|
const key = scopeKey(scope);
|
||||||
if (!enabled) { cursors.delete(key); return []; }
|
if (!enabled) { cursors.delete(key); return []; }
|
||||||
if (await toCompany(scope, worker)) return { rows: null, complete: true };
|
if (await toCompany(scope, worker)) return { rows: null, complete: true };
|
||||||
|
const changedSince = passes.changedSince(scope, key, !cursors.has(key));
|
||||||
const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId,
|
const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId,
|
||||||
issueIds: scope.issueIds } : {}, cursors.get(key) ?? null, PAGE_SIZE);
|
issueIds: scope.issueIds, ...(changedSince ? { changedSince } : {}) } : {}, cursors.get(key) ?? null, PAGE_SIZE);
|
||||||
const complete = rows.length < PAGE_SIZE;
|
const complete = rows.length < PAGE_SIZE;
|
||||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||||
passes.loaded(scope, complete);
|
passes.loaded(scope, key, complete);
|
||||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(),
|
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(),
|
||||||
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete };
|
...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete };
|
||||||
},
|
},
|
||||||
|
|||||||
Reference in new issue
Block a user