diff --git a/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts b/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts index c8b9cf500c6..9987c3283d5 100644 --- a/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts +++ b/server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts @@ -36,6 +36,7 @@ vi.mock("@paperclipai/shared/telemetry", async () => { }); import { heartbeatService } from "../services/heartbeat.ts"; +import * as runEvents from "../services/heartbeat-run-events.js"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -159,6 +160,28 @@ describeEmbeddedPostgres("heartbeat lock release on cross-agent reassignment", ( }; } + it.each(["queued", "running"] as const)("releases a %s holder after the cancellation event append fails", async (holderStatus) => { + const { coderAgentId, issueId, holderRunId, wakeupRequestId } = + await seedCrossAgentScenario({ holderStatus }); + await db.update(agents).set({ status: "running" }).where(eq(agents.id, coderAgentId)); + const append = vi.spyOn(runEvents, "appendHeartbeatRunEvent").mockRejectedValueOnce( + Object.assign(new Error("event append failed"), { code: "40P01" }), + ); + try { + const cancelled = await heartbeat.cancelRun(holderRunId); + expect(cancelled?.status).toBe("cancelled"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, holderRunId)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]) + .toMatchObject({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }); + expect((await db.select().from(agents).where(eq(agents.id, coderAgentId)))[0]?.status).toBe("idle"); + } finally { + append.mockRestore(); + } + }); + it("defers a cross-agent wake while the holder is still running and leaves the holder alone", async () => { const { coderAgentId, reviewerAgentId, issueId, holderRunId, wakeupRequestId } = await seedCrossAgentScenario({ holderStatus: "running" }); diff --git a/server/src/services/heartbeat/run-control-event-failure.test.ts b/server/src/services/heartbeat/run-control-event-failure.test.ts new file mode 100644 index 00000000000..58e260c974b --- /dev/null +++ b/server/src/services/heartbeat/run-control-event-failure.test.ts @@ -0,0 +1,52 @@ +import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { beforeAll, describe, expect, it, vi } from "vitest"; +import { createHeartbeatRunControl } from "./run-control.js"; +import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js"; +import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; +import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; + +vi.mock("../local-service-supervisor.js", async (importOriginal) => ({ + ...await importOriginal(), + terminateLocalService: vi.fn(async () => undefined), +})); + +describe.skipIf(!runControlPostgresSupport.supported)("cancellation event append failure", () => { + let db: Db; + const { fixture } = setupRunControlDatabase("paperclip-cancel-event-failure", value => { db = value; }); + beforeAll(async () => { await db.execute(sql`set client_min_messages to warning`); }); + it.each(["queued", "running"] as const)("finishes cancelling a %s run when its lifecycle event append fails", async status => { + const { company, agent } = await fixture(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, + source: "assignment", status: "claimed" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, + status, runtimeMode: "legacy", runtimeModeResolvedAt: new Date(), wakeupRequestId: wake.id }).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Release after Stop", + status: "in_progress", assigneeAgentId: agent.id, executionRunId: run.id, + executionAgentNameKey: "controlled-agent", executionLockedAt: new Date() }).returning(); + await db.update(heartbeatRuns).set({ issueId: issue.id, contextSnapshot: { issueId: issue.id } }) + .where(eq(heartbeatRuns.id, run.id)); + const deps = callbacks(db); + vi.mocked(deps.wakeQueue.releaseIssueExecution).mockImplementation(createReleaseIssueExecution({ + issueLock: createPostgresWakeQueueAdapter(db, { + resolveResponsibleUserId: async () => "operator", + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + }), + recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined }, + })); + vi.mocked(deps.appendRunEvent).mockRejectedValueOnce(new Error("Synthetic event append failure")); + + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, undefined, { + suppressImmediateRecovery: true, + })).resolves.toMatchObject({ status: "cancelled" }); + + expect(await deps.getRun(run.id)).toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0]) + .toMatchObject({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null }); + expect(deps.finalizeAgentStatus).toHaveBeenCalledExactlyOnceWith(agent.id, "cancelled", undefined, { wasFirstHeartbeat: undefined }); + expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledExactlyOnceWith(agent.id); + }); +}); diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts index fef333bc61a..81955fcd065 100644 --- a/server/src/services/heartbeat/run-control.test.ts +++ b/server/src/services/heartbeat/run-control.test.ts @@ -184,6 +184,27 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat )).toMatchObject({ status: "cancelled" }); }); + it.each(["queued", "running", "scheduled_retry"] as const)("finishes cancellation settlement for %s when the event append fails", async status => { + const { company, agent } = await fixture(); + const [wake] = await db.insert(agentWakeupRequests).values({ + companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed", + }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ + companyId: company.id, agentId: agent.id, status, wakeupRequestId: wake.id, + runtimeMode: "legacy", runtimeModeResolvedAt: new Date(), + }).returning(); + const deps = callbacks(db); + vi.mocked(deps.appendRunEvent).mockRejectedValueOnce(new Error("event storage unavailable")); + + expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" }); + expect(await deps.getRun(run.id)).toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0]) + .toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) }); + expect(deps.wakeQueue.releaseIssueExecution).toHaveBeenCalledWith(expect.objectContaining({ runId: run.id, companyId: company.id })); + expect(deps.finalizeAgentStatus).toHaveBeenCalledWith(agent.id, "cancelled", undefined, expect.any(Object)); + expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledExactlyOnceWith(agent.id); + }); + it.each([["40P01", 3], ["XX000", 1]] as const)("bounds cancellation transaction attempts for PostgreSQL %s", async (code, attempts) => { const { company, agent } = await fixture(); const run = await liveRun(company.id, agent.id); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index cde8d6acd4c..f77e8f9b013 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -1676,6 +1676,11 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont level: "warn", message: options.eventMessage ?? "run cancelled", payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, + }).catch((eventErr) => { + logger.warn( + { err: eventErr, runId: run.id }, + "failed to append run event for cancellation", + ); }); await releaseIssueExecutionAndPromote(cancelled, { suppressImmediateRecovery: options.suppressImmediateRecovery,