diff --git a/server/src/__tests__/issue-recovery-activity-hints.test.ts b/server/src/__tests__/issue-recovery-activity-hints.test.ts new file mode 100644 index 00000000000..68accc0236e --- /dev/null +++ b/server/src/__tests__/issue-recovery-activity-hints.test.ts @@ -0,0 +1,215 @@ +import { appendFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { expect, it, vi } from "vitest"; +import { agents, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; +import { trimWorkSignalStack, workSignalDebug } from "@paperclipai/db/work-signal-debug"; +import { issueService } from "../services/issues.js"; +import { describeIssueRecovery, issueRecoveryFixture } from "./issue-recovery.test-fixture.js"; +import { seedRecoveryPopulation } from "./helpers/recovery-population.js"; + +vi.hoisted(() => { process.env.PAPERCLIP_DEBUG_WORK_SIGNALS = "stranded-assignments:1,assigned-work-invariant:1"; }); + +const MINUTE = 60_000; +const measuring = Boolean(process.env.RECOVERY_ACTIVITY_REPORT); +const report = (label: string, summary: unknown) => { + if (process.env.RECOVERY_ACTIVITY_REPORT) appendFileSync(process.env.RECOVERY_ACTIVITY_REPORT, `${label} ${JSON.stringify(summary, null, 1)}\n`); +}; +const TASK_QUEUES = new Set(["stranded-assignments", "assigned-work-invariant", "resolved-dependencies", "stale-issue-locks"]); +type Recorded = { at: number; topic: string; name: string; fields: Record }; + +describeIssueRecovery("recovery checks while runs are active in a staging-shaped company", () => { + const state = issueRecoveryFixture({ meter: true }); + + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(1000, until - Date.now())); + await Promise.allSettled([...state.passes]); await immediate(); + } + } + + function record() { + const events: Recorded[] = []; + vi.spyOn(workSignalDebug, "event").mockImplementation((topic, name, fields = {}) => { + if (name === "check") events.push({ at: Date.now(), topic, name, fields }); + }); + vi.spyOn(workSignalDebug, "signal").mockImplementation((topic, hint) => { + if (!TASK_QUEUES.has(topic)) return; + const holder: { stack?: string } = {}; + const limit = Error.stackTraceLimit; + Error.stackTraceLimit = 40; + Error.captureStackTrace(holder); + Error.stackTraceLimit = limit; + const frames = trimWorkSignalStack(holder.stack, 30).filter(line => !/issue-recovery-notifications|run-dispatch-notifications|external-conversation-settlement-notifications|native-status-reassessment-notifications|execution-control-notifications|issue-review-state-notifications|mockImplementation|spy|vitest/.test(line)); + const short = (line: string) => line.replace(/^at (async )?/, "").replace(/\(?\/.*\/server\/src\//, "(").replace(/:\d+\)?$/, ")"); + const app = frames.filter(line => /services|modules|routes/.test(line)); + const caller = app.length ? app.slice(0, process.env.RECOVERY_ACTIVITY_FRAMES ? Number(process.env.RECOVERY_ACTIVITY_FRAMES) : 1).map(short).join(" < ") + : short(frames[0] ?? "unknown"); + const kind = hint === undefined || hint === "*" ? "all" : (() => { + const scope = JSON.parse(hint) as Record; + return scope.issueId ? "task" : scope.graphRunId || scope.graphIssueId ? "graph" : scope.backstop ? "backstop" : "company"; + })(); + events.push({ at: Date.now(), topic, name: "signal", fields: { kind, caller } }); + }); + return events; + } + + function summarize(events: Recorded[], seconds: number) { + const checks: Record> = {}, signals: Record = {}, scoped: Record = {}; + for (const event of events) { + if (event.name === "check") { + const byQueue = checks[event.topic] ??= {}; + byQueue[String(event.fields.wake)] = (byQueue[String(event.fields.wake)] ?? 0) + 1; + } else if (event.topic === "stranded-assignments" && (event.fields.kind === "task" || event.fields.kind === "graph")) { + const key = `${event.fields.kind} ${event.fields.caller}`; + scoped[key] = (scoped[key] ?? 0) + 1; + } else if (event.fields.kind !== "task" && event.fields.kind !== "graph") { + const key = `${event.topic} ${event.fields.kind} ${event.fields.caller}`; + signals[key] = (signals[key] ?? 0) + 1; + } + } + const perSecond = Object.fromEntries(Object.entries(checks).map(([queue, byReason]) => [queue, Object.fromEntries( + Object.entries(byReason).map(([reason, count]) => [reason, Number((count / seconds).toFixed(2))]))])); + const total = (queue: string) => Object.values(checks[queue] ?? {}).reduce((sum, count) => sum + count, 0); + return { checks, perSecond, signals, scoped, stranded: total("stranded-assignments"), assigned: total("assigned-work-invariant") }; + } + + async function provider(companyId: string, workerIds: string[]) { + const failures: unknown[] = []; + const listener = createServer(async (request, response) => { + try { + let body = ""; + for await (const chunk of request) body += chunk; + const { runId } = JSON.parse(body) as { runId: string }; + const [run] = await state.db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))); + if (!run!.contextSnapshot?.issueId) { response.end("ok"); return; } + const issueId = String(run!.contextSnapshot?.issueId); + const deadlocked = (error: unknown): boolean => typeof error === "object" && error !== null + && ((error as { code?: string }).code === "40P01" || deadlocked((error as { cause?: unknown }).cause)); + const retryDeadlock = async (write: () => Promise) => { + for (let attempt = 1; ; attempt++) { + try { return await write(); } catch (error) { if (attempt >= 3 || !deadlocked(error)) throw error; } + } + }; + await retryDeadlock(() => issueService(state.db).addComment(issueId, "Progress: done with this step", { agentId: run!.agentId, runId })); + await retryDeadlock(() => issueService(state.db).update(issueId, { status: "in_review" })); + response.end("ok"); + } catch (error) { failures.push(error); response.statusCode = 500; response.end("failed"); } + }); + await new Promise(resolve => listener.listen(0, "127.0.0.1", resolve)); + const port = (listener.address() as { port: number }).port; + const script = `fetch("http://127.0.0.1:${port}/progress",{method:"POST",body:JSON.stringify({runId:process.env.PAPERCLIP_RUN_ID})}).then(r=>{if(!r.ok)process.exitCode=1;console.log("ok")})`; + await state.db.update(agents).set({ adapterConfig: { command: process.execPath, args: ["-e", script], cwd: process.cwd() } }) + .where(inArray(agents.id, workerIds)); + return { failures, close: async () => { listener.closeAllConnections(); await new Promise(resolve => listener.close(() => resolve())); } }; + } + + async function measure(runsPerMinute: number, minutes: number, timerRunsPerMinute = 0) { + const f = await state.fixture(); + const population = await seedRecoveryPopulation(state.db, f.companyId, { agents: 50, inReview: 780, inProgress: 40, todo: 30, + recoveryActions: 12, terminated: 3 }); + const workers = population.agentIds.slice(7, 7 + 41); + const fake = await provider(f.companyId, workers); + const events = record(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await state.start({}); + await advance(10 * MINUTE); + events.length = 0; + const counts = async () => { + const [runs] = await state.db.select({ finished: sql`count(*) filter (where ${heartbeatRuns.status} = 'succeeded')::int`, + failed: sql`count(*) filter (where ${heartbeatRuns.status} = 'failed')::int`, all: sql`count(*)::int` }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.runtimeMode, "legacy"))); + const [events] = await state.db.select({ n: sql`count(*)::int` }).from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.companyId, f.companyId)); + const [comments] = await state.db.select({ n: sql`count(*)::int` }).from(issueComments) + .where(and(eq(issueComments.companyId, f.companyId), sql`${issueComments.authorAgentId} is not null`)); + return { ...runs!, events: events!.n, agentComments: comments!.n }; + }; + const workBefore = await counts(); + const before = state.meter.read(); + const startedAt = Date.now(); + const byCheck: Record = {}, settingsCallers: Record = {}; + const tally = (log: ReturnType) => { + for (const entry of log) { + const stack = entry.stack ?? ""; + const owner = stack.includes("reconcileStrandedAssignedIssues") ? "stranded-assignments" + : /assigned-work-sweep|checkIssue/.test(stack) ? "assigned-work-invariant" + : stack.includes("sweepStaleIssueLocks") ? "stale-issue-locks" : stack.includes("recovery/work.ts") ? "recovery-loads" : "other"; + byCheck[owner] = (byCheck[owner] ?? 0) + 1; + if (/from "instance_settings"/.test(entry.sql)) { + const frame = stack.split("\n").map(line => line.trim()).find(line => /server\/src\/services\//.test(line) && !/instance-settings/.test(line)) ?? "unknown"; + const key = `${owner} ${frame.replace(/^at (async )?/, "").replace(/\(?\/.*\/server\/src\//, "(").replace(/:\d+\)?$/, ")")}`; + settingsCallers[key] = (settingsCallers[key] ?? 0) + 1; + } + } + }; + if (measuring) state.meter.startLog({ stacks: true }); + let next = 12; + for (let minute = 0; minute < minutes; minute++) { + const minuteEnds = startedAt + (minute + 1) * MINUTE; + for (let started = 0; started < runsPerMinute; started++) { + const issueId = population.inReview[next++ % population.inReview.length]!; + const comment = await issueService(state.db).addComment(issueId, "Please take another pass", { userId: "recovery-user" }); + const [task] = await state.db.select({ assigneeAgentId: issues.assigneeAgentId }).from(issues).where(eq(issues.id, issueId)); + await state.heartbeat.wakeup(task!.assigneeAgentId!, { source: "automation", triggerDetail: "system", reason: "issue_commented", + payload: { issueId, commentId: comment.id, mutation: "comment" }, requestedByActorType: "user", requestedByActorId: "recovery-user", + contextSnapshot: { issueId, taskId: issueId, commentId: comment.id, wakeCommentId: comment.id, source: "issue.comment", + wakeReason: "issue_commented" } }); + await vi.advanceTimersByTimeAsync(Math.floor(MINUTE / runsPerMinute / 4)); + } + for (let timer = 0; timer < timerRunsPerMinute; timer++) { + await state.heartbeat.wakeup(workers[(minute * timerRunsPerMinute + timer) % workers.length]!, { source: "on_demand", triggerDetail: "manual", + reason: "manual", requestedByActorType: "user", requestedByActorId: "recovery-user", + contextSnapshot: { source: "scheduler", reason: "interval_elapsed", now: new Date().toISOString() } }); + } + await advance(minuteEnds - Date.now()); + await state.heartbeat.drainActiveRunExecutions(); + if (measuring) tally(state.meter.stopLog()); + if (measuring) state.meter.startLog({ stacks: true }); + } + const seconds = (Date.now() - startedAt) / 1000; + const after = state.meter.read(); + if (measuring) tally(state.meter.stopLog()); + const workAfter = await counts(); + const work = { started: workAfter.all - workBefore.all, finished: workAfter.finished - workBefore.finished, + failed: workAfter.failed - workBefore.failed, events: workAfter.events - workBefore.events, + agentComments: workAfter.agentComments - workBefore.agentComments }; + const summary = { ...summarize(events, seconds), seconds, work, + statementsPerSecond: Number(((after.statements - before.statements) / seconds).toFixed(1)), + agentWriteFailures: fake.failures.length, + statementsPerSecondBy: Object.fromEntries(Object.entries(byCheck).map(([owner, count]) => [owner, Number((count / seconds).toFixed(1))])), + instanceSettingsReads: settingsCallers }; + await fake.close(); + return summary; + } + + function expectWorkload(summary: Awaited>, taskRuns: number, tasklessRuns: number) { + expect(summary.agentWriteFailures).toBe(0); + expect(summary.work.started).toBeGreaterThanOrEqual(taskRuns + tasklessRuns); + expect(summary.work.finished).toBeGreaterThanOrEqual(taskRuns + tasklessRuns); + expect(summary.work.failed).toBeLessThanOrEqual(Math.ceil(0.1 * summary.work.started)); + expect(summary.work.agentComments).toBeGreaterThanOrEqual(taskRuns); + expect(summary.work.events).toBeGreaterThan(summary.work.finished); + } + + it("re-checks only the tasks seven runs a minute touch, among 850 open tasks", async () => { + const minutes = measuring ? 10 : 4; + const summary = await measure(7, minutes, 1); + report("7 runs/min", summary); + expectWorkload(summary, 7 * minutes, minutes); + expect(summary.checks["stranded-assignments"]?.company ?? 0).toBe(0); + expect(summary.signals).toEqual({}); + expect(summary.stranded / summary.seconds).toBeLessThan(2); + }, 1_800_000); + + it.runIf(measuring)("re-checks only the tasks fifty runs a minute touch, among 850 open tasks", async () => { + const summary = await measure(50, 6, 5); + report("50 runs/min", summary); + expectWorkload(summary, 300, 30); + expect(summary.checks["stranded-assignments"]?.company ?? 0).toBe(0); + expect(summary.signals).toEqual({}); + expect(summary.stranded / summary.seconds).toBeLessThan(3); + }, 1_800_000); +}); diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts index a63b1e3a983..b2344c7c4fc 100644 --- a/server/src/__tests__/issue-recovery-deadlines.test.ts +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -8,12 +8,15 @@ import { heartbeatService } from "../services/heartbeat.js"; import { issueService } from "../services/issues.js"; import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; import { registerIssueRecoveryWork } from "../services/recovery/work.js"; +import { changedTaskCondition } from "../services/recovery/changed-tasks.js"; import { recoveryService } from "../services/recovery/service.js"; import { ISSUE_RECOVERY_BACKSTOP_MS, ISSUE_RECOVERY_MAX_ISSUE_IDS, notifyIssueRecovery, resolveIssueGraph } from "../services/issue-recovery-notifications.js"; import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; import { notifyExecutionRunWork } from "../services/execution-control-notifications.js"; import { notifyRunDispatch } from "../services/run-dispatch-notifications.js"; +import { notifyExternalConversationSettlement } from "../services/external-conversation-settlement-notifications.js"; +import { notifyChatInboundWakeNotices } from "../services/chat-inbound-wake-notice-notifications.js"; import { promoteDeferredWorkIntents } from "../services/work-intent-outbox.js"; import { taskWatchdogService } from "../services/task-watchdogs.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; @@ -183,6 +186,68 @@ describeIssueRecovery("issue recovery production producers and deadlines", () => await graph.backlog(4000); expect(await lifecycleEvent(finished())).toBe(first); }); + it("re-checks a run's own tasks for its settlement, lease and dispatch writers, and nothing for a run without a task", async () => { + const f = await fixture(); + const graph = await runGraph(f.companyId, f.agentId); + await graph.backlog(300); + const [taskless] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "succeeded", + finishedAt: new Date(), contextSnapshot: { source: "scheduler" } }).returning({ id: heartbeatRuns.id }); + const checked: string[] = []; + const assigned: string[] = []; + await recordedRecovery({ + stranded: async scope => { checked.push(scope.issueId); }, + assignedWork: { + listSuspects: (scope, cursor, limit) => recoveryService(state.db, { enqueueWakeup: async () => null }) + .assignedWork.listSuspects(scope, cursor, limit), + checkIssue: async scope => { assigned.push(scope.issueId); }, + }, + }, { backstopMs: 60 * 60_000 }); + await settleWork({ backstopPending: true }); + const writers = (runId: string): Array<(tx: object) => Promise> => [ + tx => notifyRunDispatch(tx, { companyId: f.companyId, runId }), + tx => notifyExecutionRunWork(tx, { companyId: f.companyId, runId }), + tx => notifyExternalConversationSettlement(tx, { companyId: f.companyId, runId }), + tx => notifyChatInboundWakeNotices(tx, { companyId: f.companyId, runId }), + tx => promoteDeferredWorkIntents(tx, { companyId: f.companyId, trigger: "lock_released", issueId: null, runId }), + ]; + for (const [runId, expected] of [[graph.runId, graph.affected], [taskless.id, []]] as Array<[string, string[]]>) { + for (const writer of writers(runId)) { + checked.length = 0; assigned.length = 0; + await state.db.transaction(tx => writer(tx)); + await settleWork({ backstopPending: true }); + expect([...new Set(checked)].sort()).toEqual(expected); + expect(assigned.every(id => expected.includes(id))).toBe(true); + await vi.advanceTimersByTimeAsync(31_000); + } + } + checked.length = 0; assigned.length = 0; + await issueService(state.db).addComment(graph.siblingId, "A note on the next task", { userId: "recovery-user" }); + await settleWork({ backstopPending: true }); + expect(checked).toEqual([graph.siblingId]); + expect(assigned.every(id => id === graph.siblingId)).toBe(true); + }); + it("counts an agent's own run heartbeat as no change to its other tasks, and a resume as a change", async () => { + const f = await fixture(); + const old = new Date(Date.now() - 60 * 60_000); + const tasks = await state.db.insert(issues).values(Array.from({ length: 5 }, (_, index) => ({ companyId: f.companyId, + title: `Waiting ${index}`, status: "todo", assigneeAgentId: f.agentId, createdAt: old, updatedAt: old }))).returning({ id: issues.id }); + const since = new Date(Date.now() - 5 * 60_000); + const changed = async () => (await state.db.select({ id: issues.id }).from(issues) + .where(and(eq(issues.companyId, f.companyId), changedTaskCondition(since)))).map(row => row.id).sort(); + const heartbeatAt = new Date(); + await state.db.update(agents).set({ status: "idle", lastHeartbeatAt: heartbeatAt, updatedAt: heartbeatAt }).where(eq(agents.id, f.agentId)); + expect(await changed()).toEqual([]); + await state.db.update(agents).set({ status: "idle", updatedAt: new Date(heartbeatAt.getTime() + 1000) }).where(eq(agents.id, f.agentId)); + expect(await changed()).toEqual(tasks.map(row => row.id).sort()); + }); + it("resolves a run without a task to no tasks and a missing task to the whole company", async () => { + const f = await fixture(); + const [taskless] = await state.db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", + contextSnapshot: { source: "scheduler" } }).returning({ id: heartbeatRuns.id }); + expect(await resolveIssueGraph(state.db, { companyId: f.companyId, runId: taskless.id })).toEqual([]); + expect(await resolveIssueGraph(state.db, { companyId: f.companyId, runId: randomUUID() })).toEqual([]); + expect(await resolveIssueGraph(state.db, { companyId: f.companyId, issueId: randomUUID() })).toBeNull(); + }); it("resolves a run's task graph through index lookups, not the backlog", async () => { const f = await fixture(); const graph = await runGraph(f.companyId, f.agentId); diff --git a/server/src/services/chat-inbound-wake-notice-notifications.ts b/server/src/services/chat-inbound-wake-notice-notifications.ts index 9137e51b72f..7f38fd8b9a9 100644 --- a/server/src/services/chat-inbound-wake-notice-notifications.ts +++ b/server/src/services/chat-inbound-wake-notice-notifications.ts @@ -2,7 +2,8 @@ import { notifyExternalConversationSettlement } from "./external-conversation-se import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { InboundWakeNoticeScope } from "./chat-channels/inbound/wakeup-notices.js"; -export async function notifyChatInboundWakeNotices(tx: object, scope: InboundWakeNoticeScope, options: { notifySettlement?: boolean } = {}) { +export async function notifyChatInboundWakeNotices(tx: object, scope: InboundWakeNoticeScope & { runId?: string }, + options: { notifySettlement?: boolean } = {}) { if (options.notifySettlement !== false) await notifyExternalConversationSettlement(tx, scope); const hint = JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatInboundWakeNotices, hint); diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index b0d57b588f5..84eda427ec3 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1778,7 +1778,7 @@ export function environmentService(db: Db) { (options?.cleanupStatus !== undefined && previous.cleanupStatus !== options.cleanupStatus) || (status !== "retained" && !previous.releasedAt))) { await notifyChatInboundWakeNotices(tx, { - companyId: previous.companyId, issueId: previous.issueId ?? undefined, + companyId: previous.companyId, issueId: previous.issueId ?? undefined, runId: previous.heartbeatRunId, }); } if (previous.heartbeatRunId && ["active", "pending_cleanup"].includes(previous.status) @@ -1789,7 +1789,8 @@ export function environmentService(db: Db) { // Messages saved behind a finished run wait for its cleanup to // settle. Reconsider them through normal deferred admission. await notifyDeferredExecutionWork(tx, { companyId: previous.companyId }); - await promoteDeferredWorkIntents(tx, { companyId: previous.companyId, trigger: "lock_released", issueId: previous.issueId }); + await promoteDeferredWorkIntents(tx, { companyId: previous.companyId, trigger: "lock_released", issueId: previous.issueId, + runId: previous.heartbeatRunId }); } if (previous.creationOperationId && previous.status === "pending_cleanup" && options?.cleanupStatus === "success" && status !== "pending_cleanup") { await tx.update(sandboxCreationOperations).set({ state: "resolved", updatedAt: now }) @@ -1932,7 +1933,7 @@ export function environmentService(db: Db) { await notifyRetainedNativeSessionCleanup(tx, { companyId: run.companyId, runId: heartbeatRunId }); await notifyDeferredExecutionWork(tx, { companyId: run.companyId }); await promoteDeferredWorkIntents(tx, { companyId: run.companyId, trigger: "lock_released", - issueId: run.issueId && /^[0-9a-f-]{36}$/i.test(run.issueId) ? run.issueId : null }); + issueId: run.issueId && /^[0-9a-f-]{36}$/i.test(run.issueId) ? run.issueId : null, runId: heartbeatRunId }); } return tx.update(environmentLeases) .set({ diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 8b8e6d41617..3bfc73d77c9 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -267,7 +267,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx, { companyId: run.companyId }); + await notifyExecutionRunWork(tx, { companyId: run.companyId, runId: run.id }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index 7b0cb23dd52..b1161d584f8 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -827,7 +827,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update") : []; if (!hold) return void await continueNativeInterruption(tx as unknown as Db, { action, run, coordinator, now }); - await notifyExecutionRunWork(tx, { companyId: run.companyId }); + await notifyExecutionRunWork(tx, { companyId: run.companyId, runId: run.id }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -962,7 +962,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx, { companyId: run.companyId }); + await notifyExecutionRunWork(tx, { companyId: run.companyId, runId: run.id }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 221a9284839..74b219eb610 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -10,7 +10,7 @@ import { refreshRunStartYieldIfStale } from "../run-start-handover.js"; import { deliveredQueuedCommentIds } from "../queued-comment-delivery.js"; import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, recheckQueuedRunInteractions, withInteractionDeliveryOrigin } from "../agent-interaction-delivery.js"; import { deferredWorkIntentWait } from "../work-intent-outbox.js"; -import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; +import { notifyIssueGraphRecovery } from "../issue-recovery-notifications.js"; import { createHeartbeatTimerPolicy } from "./timer-policy.js"; import { QUEUED_RUN_AGING_MS, RECOVERY_WAKE_REASONS, createQueuedRunDispatcher, hasQueuedRunAged, queuedRunDispatchClass } from "./queued-run-dispatch-order.js"; import { reserveAutomaticRunStart } from "../recovery/automatic-run-brake.js"; @@ -587,7 +587,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const receipts = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(owned).for("update"); if (!receipts.length) return; // Register the recovery signal before the write it announces. - await notifyIssueRecovery(tx, { companyId: run.companyId }); + await notifyIssueGraphRecovery(tx, { companyId: run.companyId, runId: run.id }); await tx.update(agentWakeupRequests).set({ idempotencyKey: null, payload: sql`coalesce(${agentWakeupRequests.payload}, '{}'::jsonb) @@ -5392,8 +5392,10 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return mergedRun; } + const queuedRunId = randomUUID(); const queueOutcome = await db.transaction(async (tx) => { - await notifyRunDispatch(tx, { companyId: agent.companyId, issueId: readNonEmptyString(enrichedContextSnapshot.issueId) }); + await notifyRunDispatch(tx, { companyId: agent.companyId, issueId: readNonEmptyString(enrichedContextSnapshot.issueId), + runId: readNonEmptyString(enrichedContextSnapshot.issueId) ? undefined : queuedRunId }); await tx.execute( sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); @@ -5463,6 +5465,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const newRun = await tx .insert(heartbeatRuns) .values({ + id: queuedRunId, companyId: agent.companyId, agentId, scopeKind: readNonEmptyString(enrichedContextSnapshot.issueId) ? "issue" : "company", diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts index 5c5038d5425..37034c76bef 100644 --- a/server/src/services/issue-recovery-notifications.ts +++ b/server/src/services/issue-recovery-notifications.ts @@ -20,11 +20,8 @@ const TASK_QUEUES: DeliveryQueue[] = [DELIVERY_QUEUES.strandedAssignments, DELIV export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"; - for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.assignedWorkInvariant, DELIVERY_QUEUES.resolvedDependencies, - DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) { - await notifyDeliveryWork(transaction, queue, - queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint); - } + for (const queue of COMPANY_QUEUES) await notifyDeliveryWork(transaction, queue, companyHint); + for (const queue of TASK_QUEUES) await notifyDeliveryWork(transaction, queue, issueHint); } export function issueRecoveryBackstopHint(companyId: string) { @@ -71,5 +68,6 @@ export async function resolveIssueGraph(db: Db, scope: { companyId: string; runI and issue_id in (select id from seed) limit ${ISSUE_RECOVERY_MAX_ISSUE_IDS + 1}) ) select id::text as id from affected limit ${ISSUE_RECOVERY_MAX_ISSUE_IDS + 1}`); const issueIds = rows.map(row => row.id).sort(); - return issueIds.length && issueIds.length <= ISSUE_RECOVERY_MAX_ISSUE_IDS ? issueIds : null; + if (issueIds.length === 0) return scope.issueId ? null : []; + return issueIds.length <= ISSUE_RECOVERY_MAX_ISSUE_IDS ? issueIds : null; } diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index bc37846ba7f..f693698265d 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -161,7 +161,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick, recoveryRequestId: string | null) { - await notifyExecutionRunWork(tx, { companyId: row.run.companyId }); + await notifyExecutionRunWork(tx, { companyId: row.run.companyId, runId: row.run.id }); await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseOwner: null, leaseExpiresAt: null, nextAttemptAt: null, failureCode: "native_restart_recovery_blocked", failureDetail: { ...row.coordinator.failureDetail, reason, executorStopEvidence: "unverified" }, @@ -1932,7 +1932,7 @@ export async function claimNativeRestartRecoveries(input: { recycledProviderPids: providerProcesses.recycledPids, stderrTail: row.run.stderrExcerpt, }); - await notifyExecutionRunWork(tx, { companyId: row.run.companyId }); + await notifyExecutionRunWork(tx, { companyId: row.run.companyId, runId: row.run.id }); await tx .update(nativeRunFinalizations) .set({ diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index d9c07e2ab65..df6232eb978 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -405,7 +405,7 @@ export async function endSupersededNativeFinalizationRuns(db: Db, runIds?: reado : accepted === "succeeded" ? "succeeded" as const : accepted === "cancelled" ? "cancelled" as const : "failed" as const; await notifyNativeFinalizationWork(tx, { companyId: run.companyId, runId: run.id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); - await notifyExecutionRunWork(tx, { companyId: run.companyId }); + await notifyExecutionRunWork(tx, { companyId: run.companyId, runId: run.id }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId }); await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); @@ -626,7 +626,7 @@ async function recordRetryableFailure(input: { if (phase === "terminal_failure" && failureCode === "native_workspace_sync_out_unsafe_archive") { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId }); } - await notifyExecutionRunWork(tx, { companyId: input.run.companyId }); + await notifyExecutionRunWork(tx, { companyId: input.run.companyId, runId: input.run.id }); await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId }); await tx .update(nativeRunFinalizations) @@ -2077,7 +2077,7 @@ export async function finalizeNativeRun(input: { if (input.projectRunStatus) await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); if (input.projectRunStatus) await notifyUnsafeWorkspaceExportRunChange(tx, run); if (input.projectRunStatus) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); - await notifyExecutionRunWork(tx, { companyId: run.companyId }); + await notifyExecutionRunWork(tx, { companyId: run.companyId, runId: run.id }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId }); const [after] = await projectRun(tx); if (after && input.projectRunStatus) await settleNativeTerminalWake(tx, after); diff --git a/server/src/services/native-runtime/native-run-projection.ts b/server/src/services/native-runtime/native-run-projection.ts index c36e9b251dd..b480977eb23 100644 --- a/server/src/services/native-runtime/native-run-projection.ts +++ b/server/src/services/native-runtime/native-run-projection.ts @@ -51,7 +51,7 @@ export async function settleNativeTerminalWake(tx: Db, run: Pick= ${at}) or exists (select 1 from issue_comments c where c.company_id = ${issues.companyId} and c.issue_id = ${issues.id} and c.created_at >= ${at}) - or exists (select 1 from agents ag where ag.id = ${issues.assigneeAgentId} and ag.updated_at >= ${at}))`; + or exists (select 1 from agents ag where ag.id = ${issues.assigneeAgentId} and ag.updated_at >= ${at} + and (ag.last_heartbeat_at is null or ag.last_heartbeat_at < ag.updated_at)))`; } diff --git a/server/src/services/recovery/wake-request-claimed-settlement.ts b/server/src/services/recovery/wake-request-claimed-settlement.ts index 68ac1146803..d676ed543d5 100644 --- a/server/src/services/recovery/wake-request-claimed-settlement.ts +++ b/server/src/services/recovery/wake-request-claimed-settlement.ts @@ -64,7 +64,7 @@ export async function settleClaimedWakeRequest(db: Db, record: InProgressRecord, .where(claim).for("update"); if (!current) return { settled: false, reason: "claim_no_longer_held" }; // Work intents before the write they announce (sleep-aware background work). - await notifyExternalConversationSettlement(tx, { companyId: run.companyId }); + await notifyExternalConversationSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyStrandedAgentStatus(tx, { companyId: run.companyId, agentId: current.agentId }); const updated = await tx.update(agentWakeupRequests).set({ status: wakeStatusFor(run.status), diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts index 9926f1c9fb8..53511dde244 100644 --- a/server/src/services/recovery/work.ts +++ b/server/src/services/recovery/work.ts @@ -76,6 +76,8 @@ export function registerIssueRecoveryWork( const scopeKey = (scope: IssueRecoveryScope | undefined) => scope ? JSON.stringify([scope.companyId, scope.issueId, scope.graphRunId, scope.graphIssueId, scope.backstop]) : "*"; const isGraph = (scope: IssueRecoveryScope | undefined) => Boolean(scope?.graphRunId || scope?.graphIssueId); + const loadOrigin = (scope: IssueRecoveryScope | undefined) => !workSignalDebug.enabled ? undefined : !scope ? "reload" as const + : scope.backstop ? "backstop" as const : scope.issueId || isGraph(scope) ? "hint" as const : "company" as const; const graphs = new Map }>(); async function resolveGraph(scope: IssueRecoveryScope | undefined) { if (!scope || !isGraph(scope)) return; @@ -115,6 +117,7 @@ export function registerIssueRecoveryWork( if (workSignalDebug.enabled) workSignalDebug.event(queue, "load", { scope: key }); if (!enabled) { cursors.delete(key); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; + if (scope?.issueIds?.length === 0) return { rows: [], complete: true }; const cursor = cursors.get(key); const changedSince = passes.changedSince(scope, key, !cursor); const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( @@ -133,8 +136,7 @@ export function registerIssueRecoveryWork( if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); passes.loaded(scope, key, complete); const keepUnseen = changedSince ? { keepUnseen: true as const } : {}; - const origin = !workSignalDebug.enabled ? undefined : !scope ? "reload" as const : scope.backstop ? "backstop" as const - : scope.issueId || isGraph(scope) ? "hint" as const : "company" as const; + const origin = loadOrigin(scope); return { rows: rows.map(row => ({ ...row, dueAt: Date.now(), ...(origin ? { origin } : {}) })), complete, ...keepUnseen }; }, check: async entry => { @@ -172,22 +174,28 @@ export function registerIssueRecoveryWork( const key = scopeKey(scope); if (!enabled) { cursors.delete(key); return []; } if (await toCompany(scope, worker)) return { rows: null, complete: true }; + if (scope?.issueIds?.length === 0) return { rows: [], complete: true }; const changedSince = passes.changedSince(scope, key, !cursors.has(key)); const rows = await sweep.listSuspects(scope ? { companyId: scope.companyId, issueId: scope.issueId, issueIds: scope.issueIds, ...(changedSince ? { changedSince } : {}) } : {}, cursors.get(key) ?? null, PAGE_SIZE); const complete = rows.length < PAGE_SIZE; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); passes.loaded(scope, key, complete); + const origin = loadOrigin(scope); return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, updatedAt: row.updatedAt.getTime(), dueAt: Date.now(), - ...(scope?.backstop ? { fromBackstop: true as const } : {}) })), complete, ...(changedSince ? { keepUnseen: true as const } : {}) }; + ...(scope?.backstop ? { fromBackstop: true as const } : {}), ...(origin ? { origin } : {}) })), complete, + ...(changedSince ? { keepUnseen: true as const } : {}) }; }, check: async entry => { if (!enabled) return null; + if (workSignalDebug.enabled) workSignalDebug.event(DELIVERY_QUEUES.assignedWorkInvariant, "check", { issueId: entry.id, + companyId: entry.companyId, wake: entry.fromDeadline ? "deadline" : entry.origin ?? "load", dueAt: entry.dueAt }); let next: number | null = null, unchanged = false; await sweep.checkIssue({ companyId: entry.companyId, issueId: entry.id, onDeferredDeadline: (at, detail) => { next = next === null ? at : Math.min(next, at); unchanged ||= detail?.unchanged === true; } }); const { unchanged: _previous, fromBackstop: _source, ...rest } = entry; - return next === null ? null : { ...rest, dueAt: next, ...(unchanged ? { unchanged: true as const } : {}) }; + return next === null ? null : { ...rest, dueAt: next, ...(unchanged ? { unchanged: true as const } : {}), + ...(workSignalDebug.enabled ? { fromDeadline: true as const } : {}) }; }, }, canRun); return worker; diff --git a/server/src/services/work-intent-outbox.ts b/server/src/services/work-intent-outbox.ts index 39da2efca12..7ec9e6dee10 100644 --- a/server/src/services/work-intent-outbox.ts +++ b/server/src/services/work-intent-outbox.ts @@ -71,6 +71,7 @@ export async function promoteDeferredWorkIntents(tx: object, input: { .select({ id: budgetPolicies.id }).from(budgetPolicies).where(and(eq(budgetPolicies.companyId, input.companyId), eq(budgetPolicies.isActive, true), eq(budgetPolicies.hardStopEnabled, true))).limit(1)).length > 0 : true; const scope = input.trigger === "lock_released" && input.issueId ? { companyId: input.companyId, issueId: input.issueId } + : input.trigger === "lock_released" && input.runId ? { companyId: input.companyId, runId: input.runId } : input.trigger === "accounting_settled" && input.runId && !hardStop ? { companyId: input.companyId, runId: input.runId } : null; // Register the work signals before the write they announce. await notifyRunDispatch(tx, scope ?? { companyId: input.companyId });