diff --git a/server/src/__tests__/issue-recovery-assignments.test.ts b/server/src/__tests__/issue-recovery-assignments.test.ts index e91b0b5adfe..9f6544d824a 100644 --- a/server/src/__tests__/issue-recovery-assignments.test.ts +++ b/server/src/__tests__/issue-recovery-assignments.test.ts @@ -1,7 +1,10 @@ -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { expect, it, vi } from "vitest"; -import { heartbeatRuns, issues } from "@paperclipai/db"; +import { randomUUID } from "node:crypto"; +import { heartbeatRuns, issueRecoveryActions, issues } from "@paperclipai/db"; import { issueService } from "../services/issues.js"; +import { notifyExecutionRunWork } from "../services/execution-control-notifications.js"; +import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; describeIssueRecovery("issue recovery production producers and assignments", () => { @@ -51,4 +54,55 @@ describeIssueRecovery("issue recovery production producers and assignments", () await completed(issue.id); await finish(issue.id); await quiet(); }, 90_000); + async function scopedStart() { + const f = await fixture(); await start({}); await settled({ backstopPending: true }); + const task = async (status: "todo" | "in_progress" | "done") => (await state.db.insert(issues).values({ companyId: f.companyId, + title: `Task ${status}`, status, assigneeAgentId: f.agentId }).returning())[0]; + const run = async (issueId: string, values: Partial) => (await state.db.insert(heartbeatRuns) + .values({ companyId: f.companyId, agentId: f.agentId, contextSnapshot: { issueId }, ...values }).returning())[0]; + return { f, task, run }; + } + async function recovered(issueId: string, sourceRunId: string) { + await vi.waitFor(async () => { + const runs = await state.db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`); + const actions = await state.db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(runs.filter(row => row.id !== sourceRunId).length + actions.length).toBeGreaterThan(0); + }, { timeout: 15_000 }); + } + it("restarts a task whose run failed from that run's own hint", async () => { + const { f, task, run } = await scopedStart(); + const issue = await task("in_progress"); + const failed = await run(issue.id, { status: "failed", errorCode: "adapter_failed", finishedAt: new Date() }); + await state.db.transaction(tx => notifyExecutionRunWork(tx, { companyId: f.companyId, runId: failed.id, agentId: f.agentId })); + await recovered(issue.id, failed.id); await finish(issue.id); + }, 90_000); + it("starts the agent's queued task when its run ends and frees the agent", async () => { + const { f, task, run } = await scopedStart(); + const previous = await task("done"); + const ended = await run(previous.id, { status: "succeeded", finishedAt: new Date() }); + const next = await task("todo"); + await run(next.id, { status: "queued", invocationSource: "assignment", triggerDetail: "system" }); + await state.db.transaction(tx => notifyExecutionRunWork(tx, { companyId: f.companyId, runId: ended.id, agentId: f.agentId })); + await completed(next.id); await finish(next.id); + }, 90_000); + it("restarts a task whose run the system cancelled", async () => { + const { f, task, run } = await scopedStart(); + const issue = await task("in_progress"); + const cancelled = await run(issue.id, { status: "cancelled", finishedAt: new Date() }); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: cancelled.id })); + await recovered(issue.id, cancelled.id); await finish(issue.id); + }, 90_000); + it("restarts a task whose run vanished without a final event once its lease lapses", async () => { + const f = await fixture(); + const [issue] = await state.db.insert(issues).values({ companyId: f.companyId, title: "Lost run", status: "in_progress", + assigneeAgentId: f.agentId }).returning(); + const [lost] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", + runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: new Date(Date.now() + 5_000), + contextSnapshot: { issueId: issue.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning(); + await state.db.update(issues).set({ executionRunId: lost.id, checkoutRunId: lost.id }).where(eq(issues.id, issue.id)); + await start({}); + await recovered(issue.id, lost.id); + expect((await state.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, lost.id)))[0].status).not.toBe("running"); + await finish(issue.id); + }, 90_000); }); diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index 0908b997296..f87ab9e9564 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -3,12 +3,18 @@ import { idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { expect, it, vi } from "vitest"; -import { agents, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db"; +import { agents, budgetPolicies, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db"; import { heartbeatService } from "../services/heartbeat.js"; import { issueService } from "../services/issues.js"; import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; import { registerIssueRecoveryWork } from "../services/recovery/work.js"; -import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; +import { recoveryService } from "../services/recovery/service.js"; +import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_MAX_ISSUE_IDS, notifyIssueRecovery, resolveIssueGraph } + from "../services/issue-recovery-notifications.js"; +import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; +import { notifyExecutionRunWork } from "../services/execution-control-notifications.js"; +import { notifyRunDispatch } from "../services/run-dispatch-notifications.js"; +import { promoteDeferredWorkIntents } from "../services/work-intent-outbox.js"; import { taskWatchdogService } from "../services/task-watchdogs.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; @@ -49,11 +55,12 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => return query; }) as typeof state.db.select); await start(); await settled(); - expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8); + expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(11); expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved"); const [foreignSource] = await state.db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning(); const [foreignAction] = await state.db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id, kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning(); + await state.db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); await settled(); const before = { ...scans }; for (let index = 0; index < 100; index++) await state.db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); await new Promise(resolve => setTimeout(resolve, 100)); @@ -64,12 +71,8 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active"); expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet(); }); - it("limits an ordinary comment to its issue among one thousand open assignments", async () => { - const f = await fixture(); - const rows = await state.db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({ - companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId, - }))).returning({ id: issues.id }); - const checked: string[] = []; + async function recordedRecovery(owner: Partial[2]>, + options?: Parameters[4]) { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); state.coordinator = createDeliveryWorkCoordinator({ owner: state.db, canRun: () => true, onError: error => state.errors.push(error) }); const register = state.coordinator.register.bind(state.coordinator); @@ -78,38 +81,181 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => void pass.then(() => state.passes.delete(pass), () => state.passes.delete(pass)); return pass; } }); - // Observe the actual paged reconstruction boundary without executing one - // thousand adapters. The producer, transaction, coordinator and SQL are real; - // the other acceptance cases exercise the resulting recovery decisions. state.worker = registerIssueRecoveryWork(state.db, state.coordinator, { - stranded: async scope => { checked.push(scope.issueId); }, - orphanBlockers: async () => {}, recoveryActions: async () => {}, + stranded: async () => {}, orphanBlockers: async () => {}, recoveryActions: async () => {}, dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {}, - watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false, - }, () => true); + watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false, ...owner, + }, () => true, options); await state.worker.ready; - async function settleWork() { - for (let step = 0; step < 120; step++) { - await vi.advanceTimersByTimeAsync(1000); - await Promise.allSettled([...state.passes]); await immediate(); - if (state.coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return; - } - throw new Error("Recovery did not settle its committed work within two virtual minutes"); + } + async function settleWork(options: { backstopPending?: boolean } = {}) { + for (let step = 0; step < 120; step++) { + await vi.advanceTimersByTimeAsync(1000); + await Promise.allSettled([...state.passes]); await immediate(); + const next = state.coordinator.nextWakeAt(); + const settledNow = next === null || (options.backstopPending === true && next > Date.now() + 60_000); + if (settledNow && idleWorkSnapshot().active === 0) return; } + throw new Error("Recovery did not settle its committed work within two virtual minutes"); + } + async function runGraph(companyId: string, agentId: string) { + const [other] = await state.db.insert(agents).values({ companyId, name: "Other agent", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: {} }).returning({ id: agents.id }); + const [parent] = await state.db.insert(issues).values({ companyId, title: "Parent", status: "in_progress", assigneeAgentId: agentId }) + .returning({ id: issues.id }); + const [task] = await state.db.insert(issues).values({ companyId, title: "Run task", status: "in_progress", assigneeAgentId: agentId, + parentId: parent.id }).returning({ id: issues.id }); + const [dependent] = await state.db.insert(issues).values({ companyId, title: "Waits on the run task", status: "todo", + assigneeAgentId: other.id }).returning({ id: issues.id }); + const [sibling] = await state.db.insert(issues).values({ companyId, title: "Next task of the agent", status: "todo", + assigneeAgentId: agentId }).returning({ id: issues.id }); + await state.db.insert(issueRelations).values({ companyId, issueId: task.id, relatedIssueId: dependent.id, type: "blocks" }); + const [run] = await state.db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", finishedAt: new Date(), + contextSnapshot: { issueId: task.id } }).returning({ id: heartbeatRuns.id }); + const backlog = async (count: number) => (await state.db.insert(issues).values(Array.from({ length: count }, (_, index) => ({ + companyId, title: `Backlog ${index}`, status: "todo", assigneeAgentId: other.id, + }))).returning({ id: issues.id })).map(row => row.id); + return { runId: run.id, taskId: task.id, siblingId: sibling.id, backlog, affected: [parent.id, task.id, dependent.id].sort() }; + } + it("limits an ordinary comment to its issue among one thousand open assignments", async () => { + const f = await fixture(); + const rows = await state.db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({ + companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId, + }))).returning({ id: issues.id }); + const checked: string[] = []; + await recordedRecovery({ stranded: async scope => { checked.push(scope.issueId); } }); await settleWork(); expect(new Set(checked).size).toBe(1000); checked.length = 0; await issueService(state.db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" }); - await settleWork(); + await settleWork({ backstopPending: true }); expect(checked).toEqual([rows[0].id]); - // Company eligibility changes still reconstruct all assignments. Narrowing - // comment hints must not hide unrelated work after an operator resumes it. checked.length = 0; await state.db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); - await settleWork(); + await settleWork({ backstopPending: true }); expect(new Set(checked).size).toBe(1000); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS); + await settleWork(); await quiet(); }); + it("re-checks only the tasks a run lifecycle event can affect, whatever the backlog", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + await graph.backlog(1000); + const checked: string[] = []; + const assigned: string[] = []; + await recordedRecovery({ + stranded: async scope => { checked.push(scope.issueId); }, + assignedWork: { + listSuspects: (scope, cursor, limit) => recoveryService(state.db, { enqueueWakeup: async () => null }) + .assignedWork.listSuspects(scope, cursor, limit), + checkIssue: async scope => { assigned.push(scope.issueId); }, + }, + }, { backstopMs: 60 * 60_000 }); + await settleWork({ backstopPending: true }); + expect(new Set(checked).size).toBe(1004); + expect(checked).toContain(graph.siblingId); + async function lifecycleEvent(producer: (tx: object) => Promise, standalone?: () => Promise) { + checked.length = 0; assigned.length = 0; + const statements: string[] = [], prior = state.db.$client.options.debug; + state.db.$client.options.debug = (_connection, query) => { statements.push(query); }; + try { + if (standalone) await standalone(); else await state.db.transaction(tx => producer(tx)); + await settleWork({ backstopPending: true }); + } finally { state.db.$client.options.debug = prior; } + expect([...new Set(checked)].sort()).toEqual(graph.affected); + expect(assigned.every(id => graph.affected.includes(id))).toBe(true); + await vi.advanceTimersByTimeAsync(31_000); + return statements.length; + } + const finished = () => (tx: object) => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId }); + const first = await lifecycleEvent(finished()); + await lifecycleEvent(tx => notifyExecutionRunWork(tx, { companyId: f.companyId, runId: graph.runId, agentId: f.agentId })); + await lifecycleEvent(tx => notifyRunDispatch(tx, { companyId: f.companyId, issueId: graph.taskId })); + await lifecycleEvent(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "lock_released", issueId: graph.taskId })); + await lifecycleEvent(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "accounting_settled", runId: graph.runId })); + const recoveryActions = issueRecoveryActionService(state.db); + await lifecycleEvent(async () => {}, () => recoveryActions.upsertSourceScoped({ companyId: f.companyId, sourceIssueId: graph.taskId, + kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "run-failed", + nextAction: "Choose the next step", evidence: {} })); + await lifecycleEvent(async () => {}, () => recoveryActions.resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: graph.taskId, + status: "resolved", outcome: "restored", resolutionNote: "Recovered" })); + await graph.backlog(4000); + expect(await lifecycleEvent(finished())).toBe(first); + }); + it("resolves a run's task graph through index lookups, not the backlog", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + await graph.backlog(2000); + await state.db.execute(sql`analyze issues`); + await state.db.execute(sql`analyze heartbeat_runs`); + const [read] = await state.db.transaction(async tx => { + expect(await resolveIssueGraph(tx as unknown as typeof state.db, { companyId: f.companyId, runId: graph.runId })).toEqual(graph.affected); + return tx.execute<{ rows: number }>(sql`select (coalesce(seq_tup_read, 0) + coalesce(idx_tup_fetch, 0))::int as rows + from pg_stat_xact_user_tables where relname = 'issues'`); + }); + expect(read.rows).toBeLessThan(50); + }); + it("finds a stranded task outside the affected set within the backstop bound", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + await graph.backlog(50); + const checkedAt = new Map(); + await recordedRecovery({ stranded: async scope => { checkedAt.set(scope.issueId, Date.now()); } }); + await settleWork({ backstopPending: true }); + const fullPassAt = Date.now(); + const [unrelated] = await graph.backlog(1); + checkedAt.clear(); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + expect(checkedAt.has(unrelated)).toBe(false); + expect(state.coordinator.nextWakeAt()).toBeLessThanOrEqual(fullPassAt + ISSUE_RECOVERY_BACKSTOP_MS); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS); + await settleWork(); + expect(checkedAt.get(unrelated)! - fullPassAt).toBeLessThanOrEqual(ISSUE_RECOVERY_BACKSTOP_MS + 5_000); + checkedAt.clear(); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(checkedAt.size).toBe(0); + const [later] = await graph.backlog(1); + const eventAt = Date.now(); + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork(); + expect(checkedAt.get(later)! - eventAt).toBeLessThanOrEqual(5_000); + await quiet(); + }); + it("keeps the company pass for hold releases that can admit any task", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + const backlog = await graph.backlog(150); + const checked: string[] = []; + await recordedRecovery({ stranded: async scope => { checked.push(scope.issueId); } }); + await settleWork({ backstopPending: true }); + const everything = new Set([...graph.affected, graph.siblingId, ...backlog]); + await state.db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 1000, hardStopEnabled: true, isActive: true }); + for (const input of [{ trigger: "accounting_settled" as const, runId: graph.runId }, { trigger: "pause_lifted" as const }]) { + checked.length = 0; + await state.db.transaction(tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, ...input })); + await settleWork({ backstopPending: true }); + expect(new Set(checked)).toEqual(everything); + await vi.advanceTimersByTimeAsync(31_000); + } + }); + it("falls back to the company pass when a run affects more tasks than a hint names", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + const backlog = await graph.backlog(200); + const checked: string[] = []; + await recordedRecovery({ stranded: async scope => { checked.push(scope.issueId); } }); + await settleWork({ backstopPending: true }); + const dependents = await graph.backlog(ISSUE_RECOVERY_MAX_ISSUE_IDS); + await state.db.insert(issueRelations).values(dependents.map(id => ({ companyId: f.companyId, issueId: graph.taskId, + relatedIssueId: id, type: "blocks" as const }))); + checked.length = 0; + await state.db.transaction(tx => notifyNativeStatusReassessment(tx, { companyId: f.companyId, runId: graph.runId })); + await settleWork({ backstopPending: true }); + expect(new Set(checked)).toEqual(new Set([...graph.affected, graph.siblingId, ...backlog, ...dependents])); + }); it("checks only blocked tasks whose blockers are done, however many still wait", async () => { const f = await fixture(); const [openBlocker, doneBlocker] = await state.db.insert(issues).values([ @@ -126,28 +272,7 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => { companyId: f.companyId, issueId: doneBlocker.id, relatedIssueId: ready.id, type: "blocks" as const }, ]); const checked: string[] = []; - vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); - state.coordinator = createDeliveryWorkCoordinator({ owner: state.db, canRun: () => true, onError: error => state.errors.push(error) }); - const register = state.coordinator.register.bind(state.coordinator); - state.coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { - const pass = task.run(signal); state.passes.add(pass); - void pass.then(() => state.passes.delete(pass), () => state.passes.delete(pass)); - return pass; - } }); - state.worker = registerIssueRecoveryWork(state.db, state.coordinator, { - stranded: async () => {}, orphanBlockers: async () => {}, recoveryActions: async () => {}, - dependencies: async scope => { checked.push(scope.issueId); return { enqueueFailed: 0 }; }, locks: async () => {}, - watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false, - }, () => true); - await state.worker.ready; - async function settleWork() { - for (let step = 0; step < 120; step++) { - await vi.advanceTimersByTimeAsync(1000); - await Promise.allSettled([...state.passes]); await immediate(); - if (state.coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return; - } - throw new Error("Recovery did not settle its committed work within two virtual minutes"); - } + await recordedRecovery({ dependencies: async scope => { checked.push(scope.issueId); return { enqueueFailed: 0 }; } }); await settleWork(); expect(checked).toEqual([ready.id]); checked.length = 0; @@ -156,8 +281,10 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => expect(checked).toEqual([ready.id]); checked.length = 0; await issueService(state.db).update(openBlocker.id, { status: "done" }); - await settleWork(); + await settleWork({ backstopPending: true }); expect(new Set(checked)).toEqual(new Set([ready.id, ...waiting.map(row => row.id)])); + await vi.advanceTimersByTimeAsync(ISSUE_RECOVERY_BACKSTOP_MS); + await settleWork(); await quiet(); }); it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => { diff --git a/server/src/__tests__/issue-recovery.test-fixture.ts b/server/src/__tests__/issue-recovery.test-fixture.ts index 31a71917260..da4485f62e4 100644 --- a/server/src/__tests__/issue-recovery.test-fixture.ts +++ b/server/src/__tests__/issue-recovery.test-fixture.ts @@ -50,7 +50,7 @@ export function issueRecoveryFixture() { expect(errors).toEqual([]); }); afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); - async function start() { + async function start(options: { backstopMs?: number } = { backstopMs: 0 }) { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); tasks.clear(); @@ -63,7 +63,7 @@ export function issueRecoveryFixture() { return work; } }); }; - worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining); + worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining, options); dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining); cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining); await Promise.all([worker.ready, dispatch.ready, cancellation.ready]); @@ -77,7 +77,7 @@ export function issueRecoveryFixture() { void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass; } }); - const work = other.registerIssueStateRecoveryWork(competing, () => true); + const work = other.registerIssueStateRecoveryWork(competing, () => true, { backstopMs: 0 }); extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready; } async function fixture() { @@ -115,11 +115,12 @@ export function issueRecoveryFixture() { .where(eq(agents.id, f.agentId)); return failures; } - async function settled() { + async function settled(options: { backstopPending?: boolean } = {}) { await vi.waitFor(async () => { await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions(); - const wake = coordinator.nextWakeAt(); + const next = coordinator.nextWakeAt(); + const wake = options.backstopPending && next !== null && next > Date.now() + 60_000 ? null : next; const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))), runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) }); expect(wake, detail).toBeNull(); diff --git a/server/src/__tests__/live-recovery-work.test.ts b/server/src/__tests__/live-recovery-work.test.ts index 495b581de55..a3c1bfee699 100644 --- a/server/src/__tests__/live-recovery-work.test.ts +++ b/server/src/__tests__/live-recovery-work.test.ts @@ -104,7 +104,7 @@ describe("residual recovery production matrix", () => { const admitted = () => !draining && !standby; await heartbeat.registerResidualRecoveryWork(coordinator, admitted).ready; await heartbeat.registerActiveRunSilenceWork(coordinator, admitted).ready; - await heartbeat.registerIssueStateRecoveryWork(coordinator, admitted).ready; + await heartbeat.registerIssueStateRecoveryWork(coordinator, admitted, { backstopMs: 0 }).ready; await heartbeat.registerDeferredReceiptWork(coordinator, admitted).ready; await heartbeat.registerRunDispatchWork(coordinator, admitted).ready; await heartbeat.registerScheduledRetryCancellationWork(coordinator, admitted).ready; diff --git a/server/src/__tests__/scoped-deadline-work.test.ts b/server/src/__tests__/scoped-deadline-work.test.ts index a5a430a2410..d886f7a1fab 100644 --- a/server/src/__tests__/scoped-deadline-work.test.ts +++ b/server/src/__tests__/scoped-deadline-work.test.ts @@ -697,3 +697,27 @@ it("drops an entry that vanished from a keyed scope and keeps other companies' s expect(coordinator.nextWakeAt()).toBeNull(); expect(onError).not.toHaveBeenCalled(); }); + +it("keeps a backstop hint's deadline without holding the worker busy", async () => { + vi.useFakeTimers(); + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: vi.fn(async () => undefined) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError: vi.fn() }); + const backstopAt = Date.now() + 300_000; + const load = vi.fn(async (_scope?: string) => []); + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.externalObjectRefresh, minimumPassIntervalMs: 0, + decodeScope: hint => hint === "*" ? undefined : hint, + backstop: scope => scope === "backstop" ? backstopAt : null, + matches: () => true, load, check: async () => null, + }, () => true); + await worker.ready; + load.mockClear(); + worker.invalidate("backstop"); + await vi.advanceTimersByTimeAsync(1); + expect(load).not.toHaveBeenCalled(); + expect(coordinator.nextWakeAt()).toBe(backstopAt); + expect(worker.reconciliationSnapshot().reconciled).toBe(true); + await vi.advanceTimersByTimeAsync(300_000); + expect(load).toHaveBeenCalledWith("backstop", expect.anything()); + expect(coordinator.nextWakeAt()).toBeNull(); +}); diff --git a/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts b/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts index 96b5548e87f..b9cdf5e13d7 100644 --- a/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts +++ b/server/src/modules/run-dispatch/adapters/workspace-busy-wait.ts @@ -158,7 +158,7 @@ export async function wakeWorkspaceBusyWaitersInTx(tx: Db, input: { ); const waiters = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(waiting).for("update"); if (waiters.length === 0) return 0; - await notifyScheduledRetryDispatch(tx, { companyId: input.companyId }); + await notifyScheduledRetryDispatch(tx, { companyId: input.companyId, runId: input.runId }); const woken = await tx .update(heartbeatRuns) .set({ scheduledRetryAt: input.now, updatedAt: input.now }) diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index 471e7a00088..857ecb5ee4f 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -732,7 +732,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx, { companyId }); + await notifyRunDispatch(tx, { companyId, issueId: issue.id }); const queuedRun = await tx .insert(heartbeatRuns) .values({ @@ -811,7 +811,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx, { companyId }); + await notifyRunDispatch(tx, { companyId, issueId: issue.id }); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index b5c4fafa07b..e6a4e34589d 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -1,6 +1,6 @@ import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { assertAgentInteraction } from "./agent-interaction-policy.js"; -import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { notifyIssueRecovery, notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -290,7 +290,7 @@ export function documentService(db: Db) { .where(eq(issueDocuments.issueId, issue.id)); const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key)); - await notifyIssueRecovery(tx, { companyId: issue.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); const [document] = await tx .insert(documents) .values({ @@ -392,7 +392,7 @@ export function documentService(db: Db) { }); } - await notifyIssueRecovery(tx, { companyId: issue.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); const nextRevisionNumber = existing.latestRevisionNumber + 1; const [revision] = await tx .insert(documentRevisions) @@ -455,7 +455,7 @@ export function documentService(db: Db) { throw conflict("Document does not exist yet", { key }); } - await notifyIssueRecovery(tx, { companyId: issue.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); const [document] = await tx .insert(documents) .values({ diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 218e7e1508a..a94ac36ec91 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,5 +1,5 @@ import { notifyLiveRunRecovery, notifyLiveResourceRecovery } from "./live-run-recovery-notifications.js"; -import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; @@ -9,9 +9,14 @@ import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./deliv export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; +async function notifyExecutionAttentionRetention(transaction: object, scope: ExecutionWorkScope) { + if (scope.issueId) await notifyIssueAttentionRetentionWork(transaction, { companyId: scope.companyId, issueId: scope.issueId }); + else await notifyAttentionRetentionWork(transaction, scope); +} + export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { if (scope) { - await notifyAttentionRetentionWork(transaction, scope); + await notifyExecutionAttentionRetention(transaction, scope); await notifyRunDispatch(transaction, scope); await notifyStatusCardIssueChange(transaction, scope.companyId); await notifyDeferredExecutionEligibility(transaction, scope); @@ -36,7 +41,7 @@ export function writeExecutionControl( await notifyRunDispatch(transaction, scope); if (scope) { await notifyStatusCardIssueChange(transaction, scope.companyId); - await notifyAttentionRetentionWork(transaction, scope); + await notifyExecutionAttentionRetention(transaction, scope); await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); await notifyDeferredExecutionEligibility(transaction, scope); await notifyLiveRunRecovery(transaction, scope); diff --git a/server/src/services/external-conversation-settlement-notifications.ts b/server/src/services/external-conversation-settlement-notifications.ts index d5e4cdd506c..7a93afbf4ce 100644 --- a/server/src/services/external-conversation-settlement-notifications.ts +++ b/server/src/services/external-conversation-settlement-notifications.ts @@ -1,9 +1,10 @@ -import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { notifyIssueRecovery, notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js"; export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) { - await notifyIssueRecovery(tx, scope); + if (scope?.issueId || scope?.runId) await notifyIssueGraphRecovery(tx, scope); + else await notifyIssueRecovery(tx, scope); return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement, scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"); } diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 6dc109a8501..7f51a4b089d 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -4121,7 +4121,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId, runId: run.id }); } nativeOwnershipHeld = nativeOwnershipHeld || @@ -4725,7 +4725,8 @@ export function heartbeatService( schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, prepareChatRetry, }, () => canRun() && readTaskDrain(new Date()) === null), - registerIssueStateRecoveryWork: (coordinator: Parameters[1], canRun: () => boolean) => + registerIssueStateRecoveryWork: (coordinator: Parameters[1], canRun: () => boolean, + options?: Parameters[4]) => registerIssueRecoveryWork(db, coordinator, { stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }), orphanBlockers: recovery.reconcileUnassignedBlockingIssues, @@ -4739,7 +4740,7 @@ export function heartbeatService( checkIssue: recovery.assignedWork.checkIssue, }, schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, - }, () => canRun() && readTaskDrain(new Date()) === null), + }, () => canRun() && readTaskDrain(new Date()) === null, options), registerRunDispatchWork: (coordinator: Parameters[0], canRun: () => boolean) => registerRunDispatchWork(coordinator, { promoteDueScheduledRetries, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 18606daa274..92aa415c07d 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -5355,7 +5355,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } const queueOutcome = await db.transaction(async (tx) => { - await notifyRunDispatch(tx, { companyId: agent.companyId }); + await notifyRunDispatch(tx, { companyId: agent.companyId, issueId: readNonEmptyString(enrichedContextSnapshot.issueId) }); await tx.execute( sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index ec839348aee..c463a458a5b 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1210,7 +1210,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); - await notifyScheduledRetryDispatch(tx, { companyId: run.companyId }); + await notifyScheduledRetryDispatch(tx, { companyId: run.companyId, runId: run.id }); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ @@ -2068,7 +2068,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }; const updated = await db.transaction(async (tx) => { - await notifyScheduledRetryDispatch(tx, { companyId: scheduled.run.companyId }); + await notifyScheduledRetryDispatch(tx, { companyId: scheduled.run.companyId, runId: scheduled.run.id }); const row = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index edae7168010..e4cfaa222cc 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1557,11 +1557,15 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl ) { if (!wakeupRequestId) return; await db.transaction(async tx => { - const [current] = await tx.select({ companyId: agentWakeupRequests.companyId, agentId: agentWakeupRequests.agentId, status: agentWakeupRequests.status }) + const [current] = await tx.select({ companyId: agentWakeupRequests.companyId, agentId: agentWakeupRequests.agentId, status: agentWakeupRequests.status, + runId: agentWakeupRequests.runId, issueId: sql`${agentWakeupRequests.payload}->>'issueId'` }) .from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)).for("update"); if (!current || current.status === "cancelled") return; if (current.status !== (patch?.status ?? status) || patch?.payload !== undefined || patch?.runId !== undefined) { - await notifyExternalConversationSettlement(tx, { companyId: current.companyId }); + const runId = patch?.runId ?? current.runId; + const issueId = current.issueId && /^[0-9a-f-]{36}$/i.test(current.issueId) ? current.issueId : undefined; + await notifyExternalConversationSettlement(tx, issueId || runId + ? { companyId: current.companyId, issueId, runId: runId ?? undefined } : { companyId: current.companyId }); } await notifyStrandedAgentStatus(tx, { companyId: current.companyId, agentId: current.agentId }); await tx.update(agentWakeupRequests) @@ -2605,7 +2609,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl .returning() .then((rows) => rows[0]); - await notifyRunDispatch(tx, { companyId: run.companyId }); + await notifyRunDispatch(tx, { companyId: run.companyId, issueId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 6dd198dcbc6..5eb74902d72 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1054,7 +1054,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - ), { companyId: run.companyId }); + ), { companyId: run.companyId, runId: run.id }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1135,7 +1135,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - ), { companyId: run.companyId }); + ), { companyId: run.companyId, runId: run.id }); } catch (writeError) { if (adapterErr instanceof NativeStartupCleanupPendingError) throw adapterErr; throw writeError; diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index f98c639e8dd..f9509f33210 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,5 +1,5 @@ import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; -import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { writeExecutionControl } from "./execution-control-notifications.js"; @@ -345,7 +345,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { try { const created = await db.transaction(async (tx) => { await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); - await notifyIssueRecovery(tx, { companyId: input.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); @@ -417,7 +417,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId, issueId: input.sourceIssueId }); if (updated) return toReadModel(updated); } return existing; @@ -438,7 +438,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { Math.min(nextAttemptCount, effectiveMaxAttempts), ); const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { - await notifyIssueRecovery(tx, { companyId: input.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); if ((!input.preserveExistingOwner && existing.cause !== input.cause) || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) { @@ -493,14 +493,14 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }, { companyId: input.companyId }); + }, { companyId: input.companyId, issueId: input.sourceIssueId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } return toReadModel(exhausted); } const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { - await notifyIssueRecovery(tx, { companyId: input.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); if ((!input.preserveExistingOwner && existing.cause !== input.cause) || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) { @@ -566,7 +566,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }, { companyId: input.companyId }); + }, { companyId: input.companyId, issueId: input.sourceIssueId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -575,14 +575,14 @@ export function issueRecoveryActionService(db: DbOrTransaction) { try { const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { - await notifyIssueRecovery(tx, { companyId: input.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); return tx .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }, { companyId: input.companyId }); + }, { companyId: input.companyId, issueId: input.sourceIssueId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -632,7 +632,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { isNotNull(nativeRunFinalizations.leaseOwner), inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted"]), )).limit(1); if (waiter) await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization); - await notifyIssueRecovery(tx, { companyId: input.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); return tx @@ -646,7 +646,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }, { companyId: input.companyId }); + }, { companyId: input.companyId, issueId: input.sourceIssueId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index 42c095b35b7..ec3770b7c4d 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -1,9 +1,20 @@ -import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; /** An issue hint is reserved for local facts such as comments. Lifecycle, * graph, run and eligibility changes retain company scope because their * consequences can include ancestors and shared execution references. */ -export type IssueRecoveryScope = { companyId: string; issueId?: string }; +export type IssueRecoveryScope = { companyId: string; issueId?: string; graphRunId?: string; graphIssueId?: string; backstop?: true; + issueIds?: string[]; wholeCompany?: boolean }; + +export const ISSUE_RECOVERY_MAX_ISSUE_IDS = 100; +export const ISSUE_RECOVERY_BACKSTOP_MS = 5 * 60_000; +const UUID_PATTERN = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + +const COMPANY_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.taskWatchdogs]; +const TASK_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.assignedWorkInvariant, + DELIVERY_QUEUES.resolvedDependencies, DELIVERY_QUEUES.staleIssueLocks]; export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; @@ -14,3 +25,49 @@ export async function notifyIssueRecovery(transaction: object, scope?: IssueReco queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint); } } + +export function decodeIssueRecoveryHint(hint: string): IssueRecoveryScope | undefined { + return hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope; +} + +export function issueRecoveryScopeMatches(entry: { id: string; companyId: string }, scope?: IssueRecoveryScope) { + if (!scope) return true; + if (entry.companyId !== scope.companyId) return false; + if (scope.issueId) return entry.id === scope.issueId; + if (scope.graphRunId || scope.graphIssueId) return scope.wholeCompany === true || (scope.issueIds?.includes(entry.id) ?? false); + return true; +} + +export async function notifyIssueGraphRecovery(transaction: object, + scope: { companyId: string; runId?: string | null; issueId?: string | null }) { + if (!scope.runId && !scope.issueId) return notifyIssueRecovery(transaction, { companyId: scope.companyId }); + const companyHint = JSON.stringify({ companyId: scope.companyId }); + for (const queue of COMPANY_QUEUES) await notifyDeliveryWork(transaction, queue, companyHint); + const graphHint = JSON.stringify({ companyId: scope.companyId, graphRunId: scope.runId ?? undefined, graphIssueId: scope.issueId ?? undefined }); + for (const queue of TASK_QUEUES) { + await notifyDeliveryWork(transaction, queue, graphHint); + await notifyDeliveryWork(transaction, queue, JSON.stringify({ companyId: scope.companyId, backstop: true })); + } +} + +export async function resolveIssueGraph(db: Db, scope: { companyId: string; runId?: string; issueId?: string }): Promise { + const company = sql`${scope.companyId}::uuid`; + const run = scope.runId ? sql`${scope.runId}::uuid` : sql`null::uuid`; + const rows = await db.execute<{ id: string }>(sql`with seed(id) as ( + select id from issues where company_id = ${company} and id = ${scope.issueId ?? null}::uuid + union select issue_id from heartbeat_runs where company_id = ${company} and id = ${run} and issue_id is not null + union select native_issue_id from heartbeat_runs where company_id = ${company} and id = ${run} and native_issue_id is not null + union select issue.id from heartbeat_runs run + cross join lateral (select coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId') as saved) context + inner join issues issue on issue.company_id = run.company_id + and issue.id = case when context.saved ~* ${UUID_PATTERN} then context.saved::uuid end + where run.company_id = ${company} and run.id = ${run} + ), affected(id) as ( + select id from seed + union select parent_id from issues where company_id = ${company} and id in (select id from seed) and parent_id is not null + union (select related_issue_id from issue_relations where company_id = ${company} and type = 'blocks' + and issue_id in (select id from seed) limit ${ISSUE_RECOVERY_MAX_ISSUE_IDS + 1}) + ) select id::text as id from affected limit ${ISSUE_RECOVERY_MAX_ISSUE_IDS + 1}`); + const issueIds = rows.map(row => row.id).sort(); + return issueIds.length && issueIds.length <= ISSUE_RECOVERY_MAX_ISSUE_IDS ? issueIds : null; +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index c2cf51ce77a..0081222eb52 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -12196,7 +12196,8 @@ export function issueService(db: Db) { || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined || blockedByIssueIds !== undefined) { - await notifyRunDispatch(tx, { companyId: receiptExisting.companyId }); + await notifyRunDispatch(tx, patch.parentId !== undefined && patch.parentId !== receiptExisting.parentId + ? { companyId: receiptExisting.companyId } : { companyId: receiptExisting.companyId, issueId: id }); await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId }); await notifyActiveRunSilence(tx, { companyId: receiptExisting.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); @@ -12827,7 +12828,7 @@ export function issueService(db: Db) { await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId, issueId: id }); await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyRoutineExecutionStatus(tx, id); @@ -13238,7 +13239,7 @@ export function issueService(db: Db) { await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyRunDispatch(tx, { companyId: existing.companyId, issueId: id }); await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyRoutineExecutionStatus(tx, id); @@ -13303,7 +13304,7 @@ export function issueService(db: Db) { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id }); } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); - await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyRunDispatch(tx, { companyId: existing.companyId, issueId: id }); await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyRoutineExecutionStatus(tx, id); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index b786d4cf7f0..6d48112de2d 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -158,7 +158,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx, { companyId: current.companyId }); + await notifyExecutionRunWork(tx, { companyId: current.companyId, runId: current.id }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index cb05729dcc5..334e5880226 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -52,7 +52,7 @@ export async function materializeNativeProviderCapacityRetry(input: { executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; await notifyAttentionRetentionWork(input.tx, { companyId: input.companyId }); - await notifyScheduledRetryDispatch(input.tx, { companyId: run.companyId }); + await notifyScheduledRetryDispatch(input.tx, { companyId: run.companyId, runId: run.id }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, diff --git a/server/src/services/native-runtime/native-safe-replacement.ts b/server/src/services/native-runtime/native-safe-replacement.ts index a84c35446b6..397dd7a674a 100644 --- a/server/src/services/native-runtime/native-safe-replacement.ts +++ b/server/src/services/native-runtime/native-safe-replacement.ts @@ -775,8 +775,8 @@ export async function reconcileSafeNativeReplacements( idempotencyKey: `native-safe-replacement:${run.id}`, }) .returning(); - await notifyExecutionRunWork(tx, { companyId: run.companyId }); - await notifyScheduledRetryDispatch(tx, { companyId: run.companyId }); + await notifyExecutionRunWork(tx, { companyId: run.companyId, runId: run.id }); + await notifyScheduledRetryDispatch(tx, { companyId: run.companyId, runId: run.id }); await tx.insert(heartbeatRuns).values({ id: successorRunId, companyId: run.companyId, diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index 3fb378c7605..6189a7f73dd 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,6 +1,6 @@ import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; -import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; @@ -8,7 +8,7 @@ export type NativeStatusReassessmentScope = { companyId: string; issueId?: strin export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { await notifyStatusCardIssueChange(tx, scope.companyId); await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); - await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined); + await notifyIssueGraphRecovery(tx, scope); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/recovery/assigned-work-sweep.ts b/server/src/services/recovery/assigned-work-sweep.ts index 5559500dc9f..9317316ab80 100644 --- a/server/src/services/recovery/assigned-work-sweep.ts +++ b/server/src/services/recovery/assigned-work-sweep.ts @@ -505,7 +505,8 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { order by lr.created_at desc limit 1), false)`; } - async function listSuspects(scope: { companyId?: string; issueId?: string; createdAfter?: Date | null }, cursor: string | null, limit: number) { + async function listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[]; createdAfter?: Date | null }, + cursor: string | null, limit: number) { const at = now(); const liveRun = sql`exists (select 1 from heartbeat_runs r where r.company_id = ${issues.companyId} and (r.context_snapshot->>'issueId') = ${issues.id}::text @@ -521,6 +522,7 @@ export function assignedWorkSweep(db: Db, deps: AssignedWorkDeps) { return db.select({ id: issues.id, companyId: issues.companyId, updatedAt: issues.updatedAt }).from(issues).where(and( scope.companyId ? eq(issues.companyId, scope.companyId) : undefined, scope.issueId ? eq(issues.id, scope.issueId) : undefined, + scope.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? sql`${issues.id} > ${cursor}` : undefined, scope.createdAfter ? gte(issues.createdAt, scope.createdAfter) : undefined, inArray(issues.status, [...OPEN_STATUSES]), diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 96ba761bb7e..cd12fdcd29c 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -2832,7 +2832,7 @@ export function recoveryService( const now = new Date(); const retryAt = readProviderQuotaRetryAt(input.latestRun, now); return database.transaction(async (tx) => { - await notifyScheduledRetryDispatch(tx, { companyId: input.issue.companyId }); + await notifyScheduledRetryDispatch(tx, { companyId: input.issue.companyId, issueId: input.issue.id }); const wakeup = await tx .insert(agentWakeupRequests) .values({ @@ -3429,7 +3429,7 @@ export function recoveryService( created = Boolean(enqueuedRun); } else { scheduledRun = await db.transaction(async (tx) => { - await notifyScheduledRetryDispatch(tx, { companyId: input.issue.companyId }); + await notifyScheduledRetryDispatch(tx, { companyId: input.issue.companyId, issueId: input.issue.id }); const wakeup = await tx .insert(agentWakeupRequests) .values({ diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 00f3dbd0eb4..06a9baf803a 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -3,7 +3,8 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql, type SQL } from import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; -import type { IssueRecoveryScope } from "../issue-recovery-notifications.js"; +import { decodeIssueRecoveryHint, ISSUE_RECOVERY_BACKSTOP_MS, issueRecoveryScopeMatches, resolveIssueGraph, type IssueRecoveryScope } + from "../issue-recovery-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { recoveryActionSweepCondition } from "./blocker-stall-wakeups.js"; @@ -31,29 +32,69 @@ export function registerIssueRecoveryWork( schedulingSuppressed(): Promise; /** The assigned-work invariant (assigned-work-sweep.ts). */ assignedWork?: { - listSuspects(scope: { companyId?: string; issueId?: string }, cursor: string | null, limit: number): + listSuspects(scope: { companyId?: string; issueId?: string; issueIds?: string[] }, cursor: string | null, limit: number): Promise>; checkIssue(scope: Scope): Promise; }; }, canRun: () => boolean, + options: { backstopMs?: number } = {}, ) { + function fullPasses() { + const completedAt = new Map(); + return { + backstop: (scope: IssueRecoveryScope | undefined) => scope?.backstop + ? Math.max(completedAt.get("*") ?? 0, completedAt.get(scope.companyId) ?? 0) + (options.backstopMs ?? ISSUE_RECOVERY_BACKSTOP_MS) + : null, + loaded(scope: IssueRecoveryScope | undefined, complete: boolean) { + if (complete && !scope?.issueId && (!isGraph(scope) || scope?.wholeCompany)) completedAt.set(scope?.companyId ?? "*", Date.now()); + }, + }; + } + const reconstructionIntervalMs = (scope: IssueRecoveryScope | undefined) => scope?.backstop || scope?.graphRunId || scope?.graphIssueId ? 0 : 30_000; + const scopeKey = (scope: IssueRecoveryScope | undefined) => scope + ? JSON.stringify([scope.companyId, scope.issueId, scope.graphRunId, scope.graphIssueId, scope.backstop]) : "*"; + const isGraph = (scope: IssueRecoveryScope | undefined) => Boolean(scope?.graphRunId || scope?.graphIssueId); + const graphs = new Map }>(); + async function resolveGraph(scope: IssueRecoveryScope | undefined) { + if (!scope || !isGraph(scope)) return; + const key = scopeKey(scope); + for (const [other, cached] of graphs) if (cached.at < Date.now() - 2_000) graphs.delete(other); + let cached = graphs.get(key); + if (!cached) { + cached = { at: Date.now(), ids: resolveIssueGraph(db, { companyId: scope.companyId, runId: scope.graphRunId, issueId: scope.graphIssueId }) }; + graphs.set(key, cached); + } + const ids = await cached.ids.catch(error => { graphs.delete(key); throw error; }); + scope.issueIds = ids ?? undefined; + scope.wholeCompany = ids === null; + } + async function toCompany(scope: IssueRecoveryScope | undefined, worker: { invalidate(hint?: string): void }) { + await resolveGraph(scope); + if (!scope?.wholeCompany) return false; + worker.invalidate(JSON.stringify({ companyId: scope.companyId })); + return true; + } function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise, eligible?: SQL) { const cursors = new Map(); + const passes = fullPasses(); let enabled = false; - return registerScopedDeadlineWork(coordinator, { + const worker: ReturnType> = registerScopedDeadlineWork(coordinator, { queue, - reconstructionIntervalMs: 30_000, + reconstructionIntervalMs, + backstop: passes.backstop, beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, - decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, - matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)), + decodeScope: decodeIssueRecoveryHint, + matches: issueRecoveryScopeMatches, load: async scope => { - const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*"; + const key = scopeKey(scope); if (!enabled) { cursors.delete(key); return []; } + if (await toCompany(scope, worker)) return { rows: null, complete: true }; const cursor = cursors.get(key); const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( scope ? eq(issues.companyId, scope.companyId) : undefined, scope?.issueId ? eq(issues.id, scope.issueId) : undefined, + scope?.issueIds ? inArray(issues.id, scope.issueIds) : undefined, cursor ? gt(issues.id, cursor) : undefined, statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)), statuses ? isNull(issues.hiddenAt) : undefined, @@ -63,6 +104,7 @@ export function registerIssueRecoveryWork( )).orderBy(asc(issues.id)).limit(PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + passes.loaded(scope, complete); return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; }, check: async entry => { @@ -73,6 +115,7 @@ export function registerIssueRecoveryWork( return next === null ? null : { ...entry, dueAt: next }; }, }, canRun); + return worker; } /** Event-driven: run terminal states, hold settlements and boot mark the * company dirty; checks schedule their own grace and back-off deadlines. @@ -80,22 +123,27 @@ export function registerIssueRecoveryWork( function registerAssignedWork(sweep: NonNullable) { type Suspect = Entry & { updatedAt: number }; const cursors = new Map(); + const passes = fullPasses(); let enabled = false; - return registerScopedDeadlineWork(coordinator, { + const worker: ReturnType> = registerScopedDeadlineWork(coordinator, { queue: DELIVERY_QUEUES.assignedWorkInvariant, - reconstructionIntervalMs: 30_000, + reconstructionIntervalMs, + backstop: passes.backstop, batchSize: 25, beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, - decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, - matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)), + decodeScope: decodeIssueRecoveryHint, + matches: issueRecoveryScopeMatches, priority: entry => entry.updatedAt, mergeEntry: (pending, loaded) => ({ ...loaded, dueAt: Math.max(pending.dueAt, loaded.dueAt) }), load: async scope => { - const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*"; + const key = scopeKey(scope); if (!enabled) { cursors.delete(key); return []; } - const rows = await sweep.listSuspects(scope ?? {}, cursors.get(key) ?? null, PAGE_SIZE); + if (await toCompany(scope, worker)) return { rows: null, complete: true }; + const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId, + issueIds: scope.issueIds } : {}, cursors.get(key) ?? null, PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + passes.loaded(scope, complete); return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now() })), complete }; }, check: async entry => { @@ -106,6 +154,7 @@ export function registerIssueRecoveryWork( return next === null ? null : { ...entry, dueAt: next }; }, }, canRun); + return worker; } function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise) { const cursors = new Map(); diff --git a/server/src/services/run-cost-accounting.ts b/server/src/services/run-cost-accounting.ts index bbd5f090b10..fcf5d0062cf 100644 --- a/server/src/services/run-cost-accounting.ts +++ b/server/src/services/run-cost-accounting.ts @@ -166,8 +166,8 @@ const nativeProviderWorkNeverStarted = sql`${heartbeatRuns.runtimeMode} = 'nativ /** A run's accounting settled: an admission hold may have cleared. Held work * (queued runs, saved messages and Retries, and tasks whose start or * continuation was refused) waits passively, so wake its owners. */ -async function notifyAccountingHoldRelease(tx: object, companyId: string) { - await promoteDeferredWorkIntents(tx, { companyId, trigger: "accounting_settled" }); +async function notifyAccountingHoldRelease(tx: object, companyId: string, runId: string) { + await promoteDeferredWorkIntents(tx, { companyId, trigger: "accounting_settled", runId }); } export const ACCOUNTING_CLOSURE_TIMEOUT_MS = REMOTE_RUNNER_RECONNECT_GRACE_MS + 3 * 60_000; @@ -289,11 +289,11 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService if (run.costAccountedAt) { if (object(run.usageJson).accountingLateReceiptPending === true) { // Clearing late debt can release an admission hold too. - await notifyAccountingHoldRelease(tx, run.companyId); + await notifyAccountingHoldRelease(tx, run.companyId, run.id); return reconcileLateRunUsageReceiptInTransaction(tx, run, publications); } await notifyStrandedAgentStatus(tx, { companyId: run.companyId, agentId: run.agentId }); - await notifyAccountingHoldRelease(tx, run.companyId); + await notifyAccountingHoldRelease(tx, run.companyId, run.id); await tx.update(heartbeatRuns).set({ costAccountingPending: false }).where(eq(heartbeatRuns.id, run.id)); await queueBudgetEvaluation(tx, { companyId: run.companyId, agentId: run.agentId, projectId: text(object(object(run.usageJson).ledgerScope).projectId) }); @@ -333,7 +333,7 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService // Acknowledgement is part of the same transaction. Budget checks inside // this transaction must not mistake this receipt for missing accounting. await notifyStrandedAgentStatus(tx, { companyId: run.companyId, agentId: run.agentId }); - await notifyAccountingHoldRelease(tx, run.companyId); + await notifyAccountingHoldRelease(tx, run.companyId, run.id); await tx.update(heartbeatRuns).set({ costAccountingPending: false, costAccountedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); await tx.update(budgetReservations).set({ state: preProviderFailure ? "released" : "settled", settledAt: new Date() }) .where(and(eq(budgetReservations.companyId, run.companyId), eq(budgetReservations.runId, run.id), eq(budgetReservations.state, "held"))); @@ -426,7 +426,7 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService await queueBudgetEvaluation(tx, { companyId: run.companyId, agentId: run.agentId, projectId: text(scope.projectId) }); // Late usage reported by a later run can arrive before this charge posts. const lateDebt = usage.accountingLateReceiptPending === true; - if (!lateDebt) await notifyAccountingHoldRelease(tx, run.companyId); + if (!lateDebt) await notifyAccountingHoldRelease(tx, run.companyId, run.id); await tx.update(heartbeatRuns).set({ costAccountingPending: lateDebt, costAccountedAt: new Date(), accountingProjectionVersion: "v2", accountingLastError: lateDebt ? "late_usage_requires_reconciliation" : null, accountingLastAttemptAt: new Date(), // A complete receipt arriving later may price this charge (see usage-receipts). diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts index 36fad0e81a3..4b0f9fe6556 100644 --- a/server/src/services/run-dispatch-notifications.ts +++ b/server/src/services/run-dispatch-notifications.ts @@ -1,7 +1,7 @@ -import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { notifyIssueRecovery, notifyIssueGraphRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export type RunDispatchScope = { companyId: string }; +export type RunDispatchScope = { companyId: string; issueId?: string | null; runId?: string | null }; export async function notifyQueuedRunDispatch(transaction: object, scope?: RunDispatchScope) { await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, @@ -9,7 +9,8 @@ export async function notifyQueuedRunDispatch(transaction: object, scope?: RunDi } export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) { - await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined); + if (scope?.issueId || scope?.runId) await notifyIssueGraphRecovery(transaction, scope); + else await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined); const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; await notifyDeliveryWork(transaction, DELIVERY_QUEUES.nativeProcessSessions, hint); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.executionReplacement, hint); diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index b750d3426c4..b6ad70881c6 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -26,6 +26,7 @@ export function registerScopedDeadlineWork( admitDueWhileActive?: boolean; }; reconstructionIntervalMs?: number | ((scope: Scope | undefined) => number); + backstop?(scope: Scope | undefined): number | null; beforePass?(): Promise; decodeScope(hint: string): Scope | undefined; matches(entry: Entry, scope?: Scope): boolean; @@ -67,6 +68,12 @@ export function registerScopedDeadlineWork( decodedScopes.set(hint, scope); return scope; } + function backstopAt(hint: string) { + if (!options.backstop) return null; + try { return options.backstop(decode(hint)); } catch { return null; } + } + const holdsIdle = (hint: string) => backstopAt(hint) === null; + const gate = (hint: string) => Math.max(reconstructAfter.get(hint) ?? 0, backstopAt(hint) ?? 0); function forgetDecoded() { for (const hint of decodedScopes.keys()) if (!dirty.has(hint) && !scans.has(hint)) decodedScopes.delete(hint); } @@ -92,14 +99,14 @@ export function registerScopedDeadlineWork( function dirtyScopes() { return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint)) .filter(hint => !scans.has(hint) || scopeFailures.has(hint)) - .filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint)) + .filter(hint => gate(hint) <= Date.now() || scopeFailures.has(hint)) .map(hint => { try { return decode(hint); } catch { return undefined; } }); } function next() { if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null; let nextAt: number | null = null; const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); }; - for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0)); + for (const hint of dirty) take(Math.max(earliest, gate(hint), scopeFailures.get(hint)?.retryAt ?? 0)); const unavailable = matcher(dirtyScopes()); for (const entry of entries.values()) if (!unavailable(entry)) take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0)); @@ -119,7 +126,7 @@ export function registerScopedDeadlineWork( for (const [hint, until] of reconstructAfter) { if (until <= Date.now()) reconstructAfter.delete(hint); } - const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now()); + const ready = [...dirty].filter(hint => Math.max(gate(hint), scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now()); if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return; earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000); try { await options.beforePass?.(); } catch (error) { @@ -325,7 +332,7 @@ export function registerScopedDeadlineWork( }, // Retain scopes/deadlines for resume without retaining a settled write's // idle hold during drain. The coordinator still fences unresolved XIDs. - hasPending: async () => canRun() && dirty.size > 0, + hasPending: async () => canRun() && [...dirty].some(holdsIdle), nextRunAt: next, nextRunAtOnError: next, }); @@ -337,7 +344,7 @@ export function registerScopedDeadlineWork( return { ...worker, reconciliationSnapshot: (): DeadlineReconciliationSnapshot => ({ revision: `${ownerRevision}:${revision}`, - reconciled: !stopped && !running && !lastRunFailed && dirty.size === 0 && scans.size === 0 + reconciled: !stopped && !running && !lastRunFailed && ![...dirty].some(holdsIdle) && scans.size === 0 && failures.size === 0 && scopeFailures.size === 0, }), wake() { revision++; worker.wake(); }, diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 611f07f1a41..1ed652390a4 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -6,7 +6,7 @@ import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; import { chatConversations, chatEndpoints, heartbeatRuns, issueComments, issues, type Db } from "@paperclipai/db"; import { persistActivity, publishActivity, type ActivityPublication } from "./activity-log.js"; -export type ExternalConversationSettlementScope = { companyId: string; issueId?: string }; +export type ExternalConversationSettlementScope = { companyId: string; issueId?: string; runId?: string }; diff --git a/server/src/services/work-intent-outbox.ts b/server/src/services/work-intent-outbox.ts index 54d0eef5c39..39da2efca12 100644 --- a/server/src/services/work-intent-outbox.ts +++ b/server/src/services/work-intent-outbox.ts @@ -1,5 +1,5 @@ import { and, eq, inArray, isNotNull, isNull, ne, or, sql } from "drizzle-orm"; -import { agentWakeupRequests, issueComments, issues, type Db } from "@paperclipai/db"; +import { agentWakeupRequests, budgetPolicies, issueComments, issues, type Db } from "@paperclipai/db"; import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyDeferredWakesForIssue } from "./deferred-execution-notifications.js"; import { withQueuedCommentIdsInWakePayload } from "./issue-queued-comment-queue.js"; import { deliveredQueuedCommentIds } from "./queued-comment-delivery.js"; @@ -65,12 +65,17 @@ const DEFERRED_RETRY_BACKOFF = sql`interval '30 seconds'`; * issue by id. All notifications publish on commit. */ export async function promoteDeferredWorkIntents(tx: object, input: { - companyId: string; trigger: WorkIntentHoldTrigger; issueId?: string | null; + companyId: string; trigger: WorkIntentHoldTrigger; issueId?: string | null; runId?: string | null; }) { + const hardStop = input.trigger === "accounting_settled" && input.runId ? (await (tx as Pick) + .select({ id: budgetPolicies.id }).from(budgetPolicies).where(and(eq(budgetPolicies.companyId, input.companyId), + eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true))).limit(1)).length > 0 : true; + const scope = input.trigger === "lock_released" && input.issueId ? { companyId: input.companyId, issueId: input.issueId } + : input.trigger === "accounting_settled" && input.runId && !hardStop ? { companyId: input.companyId, runId: input.runId } : null; // Register the work signals before the write they announce. - await notifyRunDispatch(tx, { companyId: input.companyId }); + await notifyRunDispatch(tx, scope ?? { companyId: input.companyId }); await notifyDeferredExecutionEligibility(tx, { companyId: input.companyId }); - await notifyIssueRecovery(tx, { companyId: input.companyId }); + if (!scope) await notifyIssueRecovery(tx, { companyId: input.companyId }); if (input.issueId) await notifyDeferredWakesForIssue(tx, { companyId: input.companyId, issueId: input.issueId }); const db = tx as Pick; await db.update(agentWakeupRequests).set({