diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index a23dff7c4f9..e77682e455d 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -1717,6 +1717,44 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { expect(promotedRun?.agentId).toBe(peerAgentId); }); + it("delivers a person's message saved behind a system continuation cancelled at claim because its task closed", async () => { + // The claim-time closed-task cancel must leave the same never-started proof as every other + // queued-run cancel; without it task release treats the run as unfinished provider work and + // never promotes the person's message waiting behind it. + const { companyId, agentId } = await seedCompanyAndAgent(); + const [issue] = await db.insert(issues).values({ companyId, title: "Closed while its continuation waited", + status: "in_progress", assigneeAgentId: agentId }).returning(); + const { runId, wakeupRequestId } = await seedQueuedRun({ companyId, agentId, issueId: issue.id, + wakeReason: "issue_continuation_needed", invocationSource: "automation" }); + await db.update(issues).set({ status: "done", executionRunId: runId, executionLockedAt: new Date() }) + .where(eq(issues.id, issue.id)); + const [comment] = await db.insert(issueComments).values({ companyId, issueId: issue.id, authorType: "user", + authorUserId: "responsible-user", body: "Please also update the changelog" }).returning(); + const [saved] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "comment", + triggerDetail: "manual", reason: "issue_execution_deferred", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: "responsible-user", + payload: withQueuedCommentIdsInWakePayload({ issueId: issue.id, _paperclipWakeContext: { + issueId: issue.id, wakeReason: "issue_commented", wakeSource: "comment", + } }, [comment.id]), + }).returning(); + + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + + const [cancelled] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(cancelled).toMatchObject({ status: "cancelled", errorCode: "issue_closed_before_start", + resultJson: expect.objectContaining({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }) }); + expect(countExecuteCallsForRun(runId)).toBe(0); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)))[0]?.status) + .toBe("skipped"); + await vi.waitFor(async () => { + const [promoted] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, saved.id)); + expect(promoted.status).not.toBe("deferred_issue_execution"); + expect(promoted.runId).toBeTruthy(); + expect(countExecuteCallsForRun(promoted.runId!)).toBe(1); + }, { timeout: 10_000 }); + }); + it("cancels queued max-turn continuations when another continuation owns the issue lock", async () => { const { companyId, agentId } = await seedCompanyAndAgent(); const issueId = randomUUID(); diff --git a/server/src/__tests__/helpers/heartbeat-queue.ts b/server/src/__tests__/helpers/heartbeat-queue.ts index 1ad69f87f68..bea3762cb23 100644 --- a/server/src/__tests__/helpers/heartbeat-queue.ts +++ b/server/src/__tests__/helpers/heartbeat-queue.ts @@ -1,6 +1,6 @@ import type { Db } from "@paperclipai/db"; import { heartbeatRuns } from "@paperclipai/db"; -import { eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { vi, type Mock, type Mocked } from "vitest"; import { createRunDispatch } from "../../modules/run-dispatch/index.js"; import type { createWakeQueue } from "../../modules/wake-queue/index.js"; @@ -9,7 +9,7 @@ import { createHeartbeatRunPreparation } from "../../services/heartbeat/run-prep import type { HeartbeatQueueDependencies } from "../../services/heartbeat/queue.js"; type Run = typeof heartbeatRuns.$inferSelect; -type MockedCallback = "getAgent" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent" | "finalizeAgentStatus" | "getAgentInvokability" | "getWorktreeExecutionCutoff" | "applyRunDispatchPostCommitEffects" | "getSchedulingSuppression" | "canStartNewExternalBotWork" | "executeRun" | "applyWakeQueuePostCommitEffects" | "releaseIssueExecutionAndPromote" | "publishRunLifecyclePluginEvent" | "cancelRunInternal" | "cancelActiveForAgentInternal" | "resumeExecutionWaitComments" | "resumeQueuedCommentInterrupt" | "resumeSavedLegacyComments" | "sweepPendingCleanupLeases"; +type MockedCallback = "getAgent" | "setRunStatusFromLive" | "setWakeupStatus" | "appendRunEvent" | "finalizeAgentStatus" | "getAgentInvokability" | "getWorktreeExecutionCutoff" | "applyRunDispatchPostCommitEffects" | "getSchedulingSuppression" | "canStartNewExternalBotWork" | "executeRun" | "applyWakeQueuePostCommitEffects" | "releaseIssueExecutionAndPromote" | "publishRunLifecyclePluginEvent" | "cancelRunInternal" | "cancelActiveForAgentInternal" | "resumeExecutionWaitComments" | "resumeQueuedCommentInterrupt" | "resumeSavedLegacyComments" | "sweepPendingCleanupLeases"; type QueueTestDependencies = HeartbeatQueueDependencies & { [K in MockedCallback]: Mock } & { issuesSvc: Mocked; treeControlSvc: Mocked; @@ -27,8 +27,14 @@ export function createHeartbeatQueueTestDependencies(db: Db): QueueTestDependenc ...state, getAgent: vi.fn(async () => { throw new Error("Unexpected agent lookup"); }), options: {}, - setRunStatus: vi.fn(async (id, status, patch) => - db.update(heartbeatRuns).set({ ...patch, status }).where(eq(heartbeatRuns.id, id)).returning().then(rows => rows[0] ?? null)), + // Same compare-and-set contract as the run lifecycle: write only from one of fromStatuses. + setRunStatusFromLive: vi.fn(async (id, status, fromStatuses, patch) => { + const [updated] = await db.update(heartbeatRuns).set({ ...patch, status }) + .where(and(eq(heartbeatRuns.id, id), inArray(heartbeatRuns.status, fromStatuses))).returning(); + if (updated) return { run: updated, updated: true as const }; + const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)); + return { run: current ?? null, updated: false as const }; + }), setWakeupStatus: vi.fn(async () => undefined), appendRunEvent: vi.fn(async () => undefined), finalizeAgentStatus: vi.fn(async () => undefined), diff --git a/server/src/services/agent-interaction-delivery.test.ts b/server/src/services/agent-interaction-delivery.test.ts index 8b21607ec34..1514f9d7362 100644 --- a/server/src/services/agent-interaction-delivery.test.ts +++ b/server/src/services/agent-interaction-delivery.test.ts @@ -317,14 +317,12 @@ describe.skipIf(!support.supported)("pending agent interaction delivery", () => const f = await seed(); await wake(f); await restriction(f.recipient, { canReceiveAssignmentsFromAgents: false }); const callbacks = createHeartbeatQueueTestDependencies(db); - callbacks.setRunStatus.mockImplementation(async (id, status, values) => - (await db.update(heartbeatRuns).set({ ...values, status }).where(eq(heartbeatRuns.id, id)).returning())[0] ?? null); callbacks.setWakeupStatus.mockImplementation(async (id, status, values) => { if (id) await db.update(agentWakeupRequests).set({ ...values, status }).where(eq(agentWakeupRequests.id, id)); }); const queue = createHeartbeatQueue(db, callbacks); expect(await queue.claimQueuedRun(await readRun(f.run.id))).toBeNull(); - expect(callbacks.setRunStatus).toHaveBeenCalledWith(f.run.id, "cancelled", expect.objectContaining({ errorCode: "agent_interaction_delivery_denied" })); + expect(callbacks.setRunStatusFromLive).toHaveBeenCalledWith(f.run.id, "cancelled", ["queued"], expect.objectContaining({ errorCode: "agent_interaction_delivery_denied" })); expect(callbacks.executeRun).not.toHaveBeenCalled(); expect((await readRun(f.run.id)).status).toBe("cancelled"); expect(callbacks.releaseIssueExecutionAndPromote).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index dd08e34eed0..12de5b29fab 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1324,7 +1324,7 @@ export function heartbeatService( readNonEmptyString(agent.adapterConfig.externalBindingId) ?? ""); }, applyWakeQueuePostCommitEffects, - setRunStatus, + setRunStatusFromLive, setWakeupStatus, appendRunEvent, releaseIssueExecutionAndPromote, diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index 6dee76c29f3..168b0558bf6 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -62,6 +62,19 @@ describe("heartbeat queue module boundary", () => { expect(deps.executeRun).not.toHaveBeenCalled(); }); + it("claims no queued run once run admission is closed, whatever path asks (draining or yielding server)", async () => { + // Release-swap wave: an old server started a delivered continuation 3 minutes after traffic + // had moved to its dispatching replacement, through executeRun of a queued run, which no gate covered. + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.isRunAdmissionClosed = () => true; + const queue = createHeartbeatQueue(database.db, deps); + const queued = { id: "run-1", status: "queued" } as Parameters[0]; + expect(await queue.claimQueuedRun(queued)).toBeNull(); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + it("honors the process-wide run admission gate in every heartbeat instance", async () => { const database = guardedDatabase(); const deps = callbacks(database.db); @@ -497,6 +510,117 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { return { ...f, source, lease, successor, deps, queue: createHeartbeatQueue(db, deps) }; } + it.each([ + ["issue_continuation_needed", "cancelled"], ["issue_recovery_action_restored", "cancelled"], ["issue_commented", "running"], + ] as const)("does not start a %s run whose task another run already finished (expected %s)", async (wakeReason, expected) => { + // Release-swap wave t011: a delivered continuation queued while its task was open started after + // another run marked the task done, and took an extra turn on a finished task. + const f = await fixture("queued"); + await db.update(issues).set({ status: "done", executionRunId: null }).where(eq(issues.id, f.issue.id)); + // A person's message wake carries the comment it answers. + const wakeCommentId = wakeReason === "issue_commented" + ? (await db.insert(issueComments).values({ companyId: f.company.id, issueId: f.issue.id, authorAgentId: f.agent.id, + body: "One more thing" }).returning())[0].id + : undefined; + const [queued] = await db.update(heartbeatRuns).set({ responsibleUserId: "queue-test-user", + contextSnapshot: { issueId: f.issue.id, wakeReason, ...(wakeCommentId ? { wakeCommentId } : {}) } }) + .where(eq(heartbeatRuns.id, f.run.id)).returning(); + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(f.agent); + const queue = createHeartbeatQueue(db, deps); + const claimed = await queue.claimQueuedRun(queued); + if (expected === "cancelled") { + expect(claimed).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)))[0]) + .toMatchObject({ status: "cancelled", errorCode: "issue_closed_before_start" }); + } else { + // A person's message on a done task still runs (and reopens it through the normal path). + expect(claimed).toMatchObject({ id: queued.id, status: "running" }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)))[0]) + .toMatchObject({ status: "running", errorCode: null }); + } + }); + + it.each([ + { label: "still undelivered", delivered: false }, + { label: "already read by the closing run", delivered: true }, + ])("answers a person's message attached to a system continuation on a closed task once ($label)", async ({ delivered }) => { + // Admission can attach a deferred comment to a queued system continuation without changing its + // wake reason. The claim-time closed-task cancel must not consume that person's message. + const f = await fixture("queued"); + const [comment] = await db.insert(issueComments).values({ companyId: f.company.id, issueId: f.issue.id, + authorType: "user", authorUserId: "queue-test-user", body: "Can you also update the changelog?" }).returning(); + if (delivered) { + // The run that closed the task received the message as its own wake input. + const startedAt = new Date(comment.createdAt.getTime() + 1_000); + await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "succeeded", + invocationSource: "assignment", startedAt, finishedAt: new Date(startedAt.getTime() + 1_000), + contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_commented", wakeCommentIds: [comment.id] } }); + } + await db.update(issues).set({ status: "done", executionRunId: null }).where(eq(issues.id, f.issue.id)); + const [queued] = await db.update(heartbeatRuns).set({ responsibleUserId: "queue-test-user", + contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_continuation_needed", wakeCommentIds: [comment.id] } }) + .where(eq(heartbeatRuns.id, f.run.id)).returning(); + // The continuation's wake is still queued and bound to the run, as admission leaves it. + await db.update(agentWakeupRequests).set({ status: "queued", claimedAt: null, runId: queued.id }) + .where(eq(agentWakeupRequests.id, f.wake.id)); + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(f.agent); + const claimed = await createHeartbeatQueue(db, deps).claimQueuedRun(queued); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, f.agent.id)); + if (delivered) { + expect(claimed).toBeNull(); + expect(runs.find(row => row.id === queued.id)).toMatchObject({ status: "cancelled", errorCode: "issue_closed_before_start" }); + expect(runs).toHaveLength(2); + } else { + // Exactly one run answers it: this one, claimed as the person's follow-up. + expect(claimed).toMatchObject({ id: queued.id, status: "running" }); + expect(runs.filter(row => row.status !== "succeeded")).toEqual([expect.objectContaining({ id: queued.id, status: "running" })]); + } + }); + + it("keeps the result of a continuation another dispatcher already started and finished", async () => { + // Two servers overlap during a swap: both read the same queued continuation; one starts it and + // the run closes the task before the other reaches the closed-task check with its stale copy. + const f = await fixture("queued"); + const [staleQueued] = await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_continuation_needed" } }) + .where(eq(heartbeatRuns.id, f.run.id)).returning(); + const finishedAt = new Date(); + await db.update(heartbeatRuns).set({ status: "succeeded", startedAt: finishedAt, finishedAt, + resultJson: { summary: "Finished by the other server" } }).where(eq(heartbeatRuns.id, f.run.id)); + await db.update(issues).set({ status: "done", executionRunId: null }).where(eq(issues.id, f.issue.id)); + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(f.agent); + expect(await createHeartbeatQueue(db, deps).claimQueuedRun(staleQueued)).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ + status: "succeeded", errorCode: null, resultJson: { summary: "Finished by the other server" } }); + expect(deps.setWakeupStatus).not.toHaveBeenCalled(); + expect(deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled(); + }); + + it("returns the claim to the queue when run admission closes while the claim is being checked", async () => { + // SIGTERM (drain) or the hand-over yield can close admission during claimQueuedRun's awaited + // checks, after its first admission check passed. The run must stay exactly as it was. + const f = await fixture("queued"); + const [queued] = await db.update(heartbeatRuns).set({ responsibleUserId: "queue-test-user", + contextSnapshot: { issueId: f.issue.id, wakeReason: "issue_commented" } }) + .where(eq(heartbeatRuns.id, f.run.id)).returning(); + const [issueBefore] = await db.select().from(issues).where(eq(issues.id, f.issue.id)); + let admissionClosed = false; + const deps = callbacks(db); + deps.getAgent.mockResolvedValue(f.agent); + deps.isRunAdmissionClosed = () => admissionClosed; + deps.budgets.getInvocationBlock.mockImplementation(async () => { admissionClosed = true; return null; }); + expect(await createHeartbeatQueue(db, deps).claimQueuedRun(queued)).toBeNull(); + const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.id)); + expect(stored).toMatchObject({ status: "queued", startedAt: null, finishedAt: null, errorCode: null, + responsibleUserId: queued.responsibleUserId, runnerProfileJson: queued.runnerProfileJson }); + expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({ + executionRunId: issueBefore.executionRunId, executionLockedAt: issueBefore.executionLockedAt }); + expect(deps.setWakeupStatus).not.toHaveBeenCalled(); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); + it("keeps an automatic continuation queued while its legacy predecessor still owns cleanup", async () => { const { successor, lease, queue, deps, ...f } = await pendingPredecessorFixture(); expect(await queue.claimQueuedRun(successor)).toBeNull(); @@ -1011,7 +1135,7 @@ describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe(wake.status); expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0].status).toBe("todo"); expect(deps.executeRun).not.toHaveBeenCalled(); - expect(deps.setRunStatus).not.toHaveBeenCalled(); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); }); it("admits the same queued input after the external connection is repaired", async () => { diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index fa0ff7b8b59..e55b996a0aa 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -184,7 +184,7 @@ import { WORKSPACE_WORKTREE_REQUIRES_PROJECT_REMEDIATION, } from "../execution-workspace-policy.js"; import { readContinuationAttempt } from "../recovery/index.js"; -import { allowsDependencyBlockedWake, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js"; +import { allowsDependencyBlockedWake, extractWakeCommentIds, wakeChatConversationsConflict } from "../../modules/run-dispatch/index.js"; import { withAgentStartLock } from "../agent-start-lock.js"; import { evaluateAgentInvokability, @@ -192,7 +192,7 @@ import { type AgentOrgRow, } from "../agent-invokability.js"; import { supportsUnscopedTimerHeartbeat } from "../heartbeat-policy.js"; -import { isUuidLike } from "@paperclipai/shared"; +import { ISSUE_BLOCKER_STALLED_WAKE_REASON, isUuidLike } from "@paperclipai/shared"; import { retryChatControlAdmission } from "../chat-control-admission-retry.js"; import type { HeartbeatServiceOptions } from "../heartbeat.js"; @@ -222,7 +222,7 @@ export interface HeartbeatQueueDependencies extends Pick, Pick, - Pick { + Pick { sweepPendingCleanupLeases: ReturnType["sweepPendingCleanupLeases"]; options: Pick; issuesSvc: Pick, "getDependencyReadiness" | "listDependencyReadiness" | "updateForCompany">; @@ -412,12 +412,15 @@ export async function claimDueTimerHeartbeatForAgent( /** Queue admission and dispatch share the service lifecycle callbacks and execution ownership. */ // Wake admissions that have not committed their wake and run yet. Process-wide, like // the pending-wake set: a server replacement loses these, and only these, pending wakes. +/** Rolls back a claim whose server closed run admission while it was being taken. */ +class RunAdmissionClosedDuringClaim extends Error {} + const unsavedWakeAdmissions = new Set(); export function unsavedWakeAdmissionCount() { return unsavedWakeAdmissions.size; } export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDependencies) { const { - setRunStatus, + setRunStatusFromLive, setWakeupStatus, appendRunEvent, releaseIssueExecutionAndPromote, @@ -469,6 +472,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const { parseHeartbeatPolicy, getHeartbeatDailyCapBlock, hasActionableTimerWork, nextTimerHeartbeatAt } = createHeartbeatTimerPolicy(db); + // claimQueuedRun works from the caller's copy of a queued run. Another + // dispatcher (an overlapping server during a swap) may have started and even + // finished that run since, so every cancellation it makes is a + // compare-and-set from "queued": it never overwrites a newer outcome. + async function cancelRunIfStillQueued(runId: string, patch: Partial) { + const write = await setRunStatusFromLive(runId, "cancelled", ["queued"], patch); + return write.updated ? write.run : null; + } + async function cancelQueuedRunForHeartbeatDailyCap( run: typeof heartbeatRuns.$inferSelect, dailyCapBlock: NonNullable< @@ -478,7 +490,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const now = new Date(); const reason = "Cancelled because the agent reached a per-day heartbeat budget cap before adapter invocation"; - const cancelled = await setRunStatus(run.id, "cancelled", { + const cancelled = await cancelRunIfStillQueued(run.id, { finishedAt: now, error: reason, errorCode: dailyCapBlock.reason, @@ -869,6 +881,76 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } } + // Run admission can close (SIGTERM drain, or the hand-over yield) while + // claimQueuedRun awaits its checks, after its first admission check passed. + // Checked again inside the claim's transaction, right after the write that + // moves the run to running: throwing rolls the whole claim back, so the run, + // its wake and the task's execution lock stay exactly as they were, queued + // for the serving server. + function confirmRunAdmissionStillOpen(claimedRun: unknown) { + if (claimedRun && isRunAdmissionClosed()) throw new RunAdmissionClosedDuringClaim(); + } + + /** System continuations that only make sense for an open task. */ + const SYSTEM_CONTINUATION_WAKE_REASONS = new Set(["issue_recovery_action_restored", "issue_continuation_needed", + "issue_assignment_recovery", ISSUE_BLOCKER_STALLED_WAKE_REASON]); + + /** + * Cancels a system continuation whose task closed while it was queued, + * unless it carries a person's message that no run of this agent has + * received yet (admission can attach deferred comments to a queued system + * continuation without changing its wake reason). Its wake and run are + * locked in the order admission's coalescing uses while the message check + * reads them. The cancel itself goes through the run lifecycle's + * compare-and-set, so it saves the same never-started proof as every other + * queued-run cancel (task release needs it to promote messages saved behind + * this run), and it is fenced on the row version the check saw: a message + * attached after the check changes the run, the cancel matches nothing, and + * the check runs again. + */ + async function cancelClosedTaskContinuation(run: typeof heartbeatRuns.$inferSelect, issueId: string) { + for (let attempt = 0; attempt < 3; attempt += 1) { + const decision = await checkClosedTaskContinuation(run, issueId); + if (decision.kind !== "cancel") return decision; + const write = await setRunStatusFromLive(run.id, "cancelled", ["queued"], { + finishedAt: new Date(), error: "The task was closed before this continuation started.", + errorCode: "issue_closed_before_start", + }, undefined, sql`${heartbeatRuns}.xmin::text = ${decision.rowVersion}`); + if (write.updated && write.run) return { kind: "cancelled" as const, run: write.run }; + if (write.run?.status !== "queued") return { kind: "not_queued" as const }; + } + // Still changing under us: leave it queued; the next dispatch pass checks again. + return { kind: "still_queued" as const }; + } + + async function checkClosedTaskContinuation(run: typeof heartbeatRuns.$inferSelect, issueId: string) { + return db.transaction(async (tx) => { + if (run.wakeupRequestId) await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, run.wakeupRequestId), eq(agentWakeupRequests.companyId, run.companyId), + )).for("update"); + const [locked] = await tx.select({ run: heartbeatRuns, rowVersion: sql`${heartbeatRuns}.xmin::text` }) + .from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"), + )).for("update"); + if (!locked) return { kind: "not_queued" as const }; + const current = locked.run; + const context = parseObject(current.contextSnapshot); + const commentIds = [...new Set([...extractWakeCommentIds(context), ...queuedCommentIdsFromRunContext(context), + readNonEmptyString(context.wakeCommentId), readNonEmptyString(context.commentId)] + .filter((id): id is string => Boolean(id)))]; + if (commentIds.length > 0) { + const personComments = await tx.select({ id: issueComments.id }).from(issueComments).where(and( + eq(issueComments.companyId, current.companyId), eq(issueComments.issueId, issueId), + inArray(issueComments.id, commentIds), eq(issueComments.authorType, "user"), + isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt))); + const delivered = await deliveredQueuedCommentIds(tx as unknown as Db, { companyId: current.companyId, issueId, + agentId: current.agentId, commentIds: personComments.map(comment => comment.id) }); + if (personComments.some(comment => !delivered.has(comment.id))) return { kind: "carries_person_message" as const }; + } + return { kind: "cancel" as const, rowVersion: locked.rowVersion }; + }); + } + async function claimQueuedRun( run: typeof heartbeatRuns.$inferSelect, companyAgents?: AgentOrgRow[], @@ -876,12 +958,40 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende deferredCancellations?: Array<() => Promise>, ) { if (run.status !== "queued") return run; + // Every start path ends here (dispatch passes, executeRun of a queued run, + // a delivered continuation). A server whose run admission is closed (it + // received SIGTERM and is draining) or that yields new starts to a + // replacement already dispatching leaves the run queued for that server: + // anything it started now would be cut off by its own exit. + const refreshing = refreshRunAdmission(); + if (refreshing) await refreshing; + if (isRunAdmissionClosed()) return null; + // A system continuation queued while its task was open has nothing left to + // do once another run finished or cancelled the task before this one + // started. A person's message on a done task is not one of these reasons. + const systemContinuationReason = readNonEmptyString(parseObject(run.contextSnapshot).wakeReason); + const continuedIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + if (systemContinuationReason && SYSTEM_CONTINUATION_WAKE_REASONS.has(systemContinuationReason) && continuedIssueId) { + const [target] = await db.select({ status: issues.status }).from(issues) + .where(and(eq(issues.companyId, run.companyId), eq(issues.id, continuedIssueId))); + if (target && ["done", "cancelled"].includes(target.status)) { + const closed = await cancelClosedTaskContinuation(run, continuedIssueId); + if (closed.kind === "cancelled") { + const cancelled = closed.run; + await setWakeupStatus(run.wakeupRequestId, "skipped", { finishedAt: cancelled.finishedAt ?? new Date(), error: "issue_closed_before_start" }); + await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: true, deferredPostCommitEffects }); + } + // A person's undelivered message rides on this run: it runs as their + // follow-up, which may reopen the task through the normal path. + if (closed.kind !== "carries_person_message") return null; + } + } const interactionDelivery = await recheckQueuedRunInteractions(db, run); if (interactionDelivery.run.status !== "queued") return null; run = { ...run, contextSnapshot: interactionDelivery.run.contextSnapshot, resultJson: interactionDelivery.run.resultJson, updatedAt: interactionDelivery.run.updatedAt }; if (interactionDelivery.denied) { - const cancelled = await setRunStatus(run.id, "cancelled", { + const cancelled = await cancelRunIfStillQueued(run.id, { finishedAt: new Date(), error: "Queued agent input is no longer permitted.", errorCode: INTERACTION_DELIVERY_DENIED, }); @@ -940,7 +1050,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const reason = "Message did not @mention the bot"; // This runs under the agent start lock. Do not use cancelRunInternal, // which recursively starts the next run and reacquires that lock. - const cancelled = await setRunStatus(run.id, "cancelled", { + const cancelled = await cancelRunIfStillQueued(run.id, { finishedAt: now, error: reason, errorCode: "chat_mention_required", @@ -1487,6 +1597,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ) .returning(); await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); + confirmRunAdmissionStillOpen(claimedRun); return claimedRun ? { kind: "claimed" as const, run: claimedRun } : { kind: "stale" as const, run: null }; @@ -1598,12 +1709,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ) .returning(); await bindClaimedIssueExecution(tx as unknown as Db, issueClaim.ownsIssue, claimedRun); + confirmRunAdmissionStillOpen(claimedRun); return claimedRun ? { kind: "claimed" as const, run: claimedRun } : { kind: "stale" as const, run: null }; }) .catch((error) => { - if (isExternalChatWaitAuthorizationContention(error)) + if (isExternalChatWaitAuthorizationContention(error) || error instanceof RunAdmissionClosedDuringClaim) return { kind: "stale" as const, run: null }; throw error; }) @@ -1658,9 +1770,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const predecessor = await lockIssueExecutionClaim(tx); if (predecessor.blocked) return null; } - return claimQueuedNativeReviewRun(tx, { - run, claimedAt, claimValues, - agentNameKey: normalizeAgentNameKey(agent.name), + return tx.transaction(async (reviewTx) => { + const reviewClaim = await claimQueuedNativeReviewRun(reviewTx as unknown as Db, { + run, claimedAt, claimValues, + agentNameKey: normalizeAgentNameKey(agent.name), + }); + confirmRunAdmissionStillOpen(reviewClaim); + return reviewClaim; }); } return tx.transaction(async (claimTx) => { @@ -1679,8 +1795,12 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"), )).returning().then((rows) => rows[0] ?? null); await bindClaimedIssueExecution(claimTx as unknown as Db, issueClaim.ownsIssue, claimedRun); + confirmRunAdmissionStillOpen(claimedRun); return claimedRun; }); + }).catch((error: unknown) => { + if (error instanceof RunAdmissionClosedDuringClaim) return null; + throw error; }); if (!claimed) return null; @@ -1831,7 +1951,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const now = new Date(); const reason = "Cancelled because issue dependencies are still blocked; Paperclip will wake the assignee when blockers resolve"; - const cancelled = await setRunStatus(run.id, "cancelled", { + const cancelled = await cancelRunIfStillQueued(run.id, { finishedAt: now, error: reason, errorCode: "issue_dependencies_blocked", diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index d2f4fe09b02..bab1110db5a 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -72,6 +72,7 @@ import { ne, or, sql, + type SQL, } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { @@ -173,7 +174,7 @@ export interface HeartbeatRecoveryDependencies extends Pick Promise; setRunStatus: (runId: string, status: string, patch?: Partial) => Promise; setRunStatusFromLive: (runId: string, status: string, fromStatuses: string[], patch?: Partial, - failureReport?: undefined, cancellationCondition?: undefined, options?: { onlyWhileNativeLeaseLapsed?: true }) => Promise<{ + failureReport?: undefined, cancellationCondition?: SQL, options?: { onlyWhileNativeLeaseLapsed?: true }) => Promise<{ updated: boolean; run: HeartbeatRun | null; }>;