From 6f2ea3b4d5bb56443160211d34e99cf5c92a75ca Mon Sep 17 00:00:00 2001 From: "commitperclip[bot]" <284812584+commitperclip[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:58:29 -0600 Subject: [PATCH] fix: Complete run cancellation when lifecycle event append fails (#20215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > [!NOTE] > Opened by Prospector's automated issue-fix pipeline. A maintainer reviews and decides; nothing here merges on its own. > [!WARNING] > This change crosses a trust boundary. See Risks. ## Thinking Path - Issue #20116 reports a defect in paperclipai/paperclip. - Root cause, as the fixing agent read it: Cancellation awaited the lifecycle event append after cancelling the run and wakeup. An append failure rejected the stop before issue release, agent finalization, and queue advancement. - This pull request: Complete run cancellation when lifecycle event append fails ## What Changed - `server/src/services/heartbeat/run-control.ts`: Log cancellation event append failures and continue essential cleanup, matching lease-release terminalization. - `server/src/services/heartbeat/run-control-event-failure.test.ts`: Cover queued and running cancellation with an injected append failure and the real issue-release adapter. Confirmed regression failures before the fix; regression tests, neighboring tests, and final typecheck pass. ## Verification - Reproducing test(s), added in this PR: `server/src/services/heartbeat/run-control-event-failure.test.ts`, `server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts`, `server/src/services/heartbeat/run-control.test.ts` - They fail on the base commit and pass with the fix, each twice in a fresh sandbox. - 3 independent attempts wrote 3 reproductions; this fix passes every one of them. - The repository's compile command passes with the fix applied. - 9 existing test file(s) near the change passed. - The full test suite passes except for the 16 file(s) that already fail on the base commit. ## Risks - Written by an automated pipeline, not a person: review it as you would an outside contribution. - Risk tier of the touched paths: 2. - It crosses a trust boundary. Check each crossing against what the issue asked for, and against who asked: - logging (server/src/services/heartbeat/run-control.ts:1665): Logs the event-append error and run ID to the server log when cancellation event recording fails. ## Model Used - codex:gpt-6.1-sol, opus, sonnet (Anthropic Claude, via Claude Code; OpenAI, via the Codex CLI), writing and checking the change in an isolated sandbox. - opus (Anthropic Claude, via Claude Code), reviewing it independently. ## Linked Issues Fixes #20116 The reviewer found the base already handles: - A closed request socket produces a misleading PATCH 200 log that hides the failure. (server/src/middleware/logger.ts:89–91 detects aborted or unfinished responses and formats 'request aborted'; lines 150 and 157–158 log these at warn level with that message instead of the default status-based success message.) A search of paperclipai/paperclip's open and closed pull requests found no other pull request that names #20116. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [ ] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with 'Fixes: #' / 'Closes #' / 'Refs #' OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub '#NNN' / 'github.com/paperclipai/paperclip' URLs) - [ ] My branch name describes the change (e.g. 'docs/...', 'fix/...') and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: commitperclip[bot] <284812584+commitperclip[bot]@users.noreply.github.com> --- ...tbeat-lock-release-on-reassignment.test.ts | 23 ++++++++ .../run-control-event-failure.test.ts | 52 +++++++++++++++++++ .../services/heartbeat/run-control.test.ts | 21 ++++++++ server/src/services/heartbeat/run-control.ts | 5 ++ 4 files changed, 101 insertions(+) create mode 100644 server/src/services/heartbeat/run-control-event-failure.test.ts diff --git a/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts b/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts index c8b9cf500c6..9987c3283d5 100644 --- a/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts +++ b/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts @@ -36,6 +36,7 @@ vi.mock("@paperclipai/shared/telemetry", async () => { }); import { heartbeatService } from "../services/heartbeat.ts"; +import * as runEvents from "../services/heartbeat-run-events.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -159,6 +160,28 @@ describeEmbeddedPostgres("heartbeat lock release on cross-agent reassignment", ( }; } + it.each(["queued", "running"] as const)("releases a %s holder after the cancellation event append fails", async (holderStatus) => { + const { coderAgentId, issueId, holderRunId, wakeupRequestId } = + await seedCrossAgentScenario({ holderStatus }); + await db.update(agents).set({ status: "running" }).where(eq(agents.id, coderAgentId)); + const append = vi.spyOn(runEvents, "appendHeartbeatRunEvent").mockRejectedValueOnce( + Object.assign(new Error("event append failed"), { code: "40P01" }), + ); + try { + const cancelled = await heartbeat.cancelRun(holderRunId); + expect(cancelled?.status).toBe("cancelled"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, holderRunId)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]) + .toMatchObject({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }); + expect((await db.select().from(agents).where(eq(agents.id, coderAgentId)))[0]?.status).toBe("idle"); + } finally { + append.mockRestore(); + } + }); + it("defers a cross-agent wake while the holder is still running and leaves the holder alone", async () => { const { coderAgentId, reviewerAgentId, issueId, holderRunId, wakeupRequestId } = await seedCrossAgentScenario({ holderStatus: "running" }); diff --git a/server/src/services/heartbeat/run-control-event-failure.test.ts b/server/src/services/heartbeat/run-control-event-failure.test.ts new file mode 100644 index 00000000000..58e260c974b --- /dev/null +++ b/server/src/services/heartbeat/run-control-event-failure.test.ts @@ -0,0 +1,52 @@ +import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { beforeAll, describe, expect, it, vi } from "vitest"; +import { createHeartbeatRunControl } from "./run-control.js"; +import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js"; +import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; +import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; + +vi.mock("../local-service-supervisor.js", async (importOriginal) => ({ + ...await importOriginal(), + terminateLocalService: vi.fn(async () => undefined), +})); + +describe.skipIf(!runControlPostgresSupport.supported)("cancellation event append failure", () => { + let db: Db; + const { fixture } = setupRunControlDatabase("paperclip-cancel-event-failure", value => { db = value; }); + beforeAll(async () => { await db.execute(sql`set client_min_messages to warning`); }); + it.each(["queued", "running"] as const)("finishes cancelling a %s run when its lifecycle event append fails", async status => { + const { company, agent } = await fixture(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, + source: "assignment", status: "claimed" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + status, runtimeMode: "legacy", runtimeModeResolvedAt: new Date(), wakeupRequestId: wake.id }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Release after Stop", + status: "in_progress", assigneeAgentId: agent.id, executionRunId: run.id, + executionAgentNameKey: "controlled-agent", executionLockedAt: new Date() }).returning(); + await db.update(heartbeatRuns).set({ issueId: issue.id, contextSnapshot: { issueId: issue.id } }) + .where(eq(heartbeatRuns.id, run.id)); + const deps = callbacks(db); + vi.mocked(deps.wakeQueue.releaseIssueExecution).mockImplementation(createReleaseIssueExecution({ + issueLock: createPostgresWakeQueueAdapter(db, { + resolveResponsibleUserId: async () => "operator", + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + }), + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + })); + vi.mocked(deps.appendRunEvent).mockRejectedValueOnce(new Error("Synthetic event append failure")); + + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, undefined, { + suppressImmediateRecovery: true, + })).resolves.toMatchObject({ status: "cancelled" }); + + expect(await deps.getRun(run.id)).toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0]) + .toMatchObject({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }); + expect(deps.finalizeAgentStatus).toHaveBeenCalledExactlyOnceWith(agent.id, "cancelled", undefined, { wasFirstHeartbeat: undefined }); + expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledExactlyOnceWith(agent.id); + }); +}); diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts index fef333bc61a..81955fcd065 100644 --- a/server/src/services/heartbeat/run-control.test.ts +++ b/server/src/services/heartbeat/run-control.test.ts @@ -184,6 +184,27 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat )).toMatchObject({ status: "cancelled" }); }); + it.each(["queued", "running", "scheduled_retry"] as const)("finishes cancellation settlement for %s when the event append fails", async status => { + const { company, agent } = await fixture(); + const [wake] = await db.insert(agentWakeupRequests).values({ + companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed", + }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ + companyId: company.id, agentId: agent.id, status, wakeupRequestId: wake.id, + runtimeMode: "legacy", runtimeModeResolvedAt: new Date(), + }).returning(); + const deps = callbacks(db); + vi.mocked(deps.appendRunEvent).mockRejectedValueOnce(new Error("event storage unavailable")); + + expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" }); + expect(await deps.getRun(run.id)).toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect(deps.wakeQueue.releaseIssueExecution).toHaveBeenCalledWith(expect.objectContaining({ runId: run.id, companyId: company.id })); + expect(deps.finalizeAgentStatus).toHaveBeenCalledWith(agent.id, "cancelled", undefined, expect.any(Object)); + expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledExactlyOnceWith(agent.id); + }); + it.each([["40P01", 3], ["XX000", 1]] as const)("bounds cancellation transaction attempts for PostgreSQL %s", async (code, attempts) => { const { company, agent } = await fixture(); const run = await liveRun(company.id, agent.id); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index cde8d6acd4c..f77e8f9b013 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -1676,6 +1676,11 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont level: "warn", message: options.eventMessage ?? "run cancelled", payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, + }).catch((eventErr) => { + logger.warn( + { err: eventErr, runId: run.id }, + "failed to append run event for cancellation", + ); }); await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: options.suppressImmediateRecovery,