mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 11:59:26 +01:00
fix: Complete run cancellation when lifecycle event append fails (#20215)
> [!NOTE] > Opened by Prospector's automated issue-fix pipeline. A maintainer reviews and decides; nothing here merges on its own. > [!WARNING] > This change crosses a trust boundary. See Risks. ## Thinking Path - Issue #20116 reports a defect in paperclipai/paperclip. - Root cause, as the fixing agent read it: Cancellation awaited the lifecycle event append after cancelling the run and wakeup. An append failure rejected the stop before issue release, agent finalization, and queue advancement. - This pull request: Complete run cancellation when lifecycle event append fails ## What Changed - `server/src/services/heartbeat/run-control.ts`: Log cancellation event append failures and continue essential cleanup, matching lease-release terminalization. - `server/src/services/heartbeat/run-control-event-failure.test.ts`: Cover queued and running cancellation with an injected append failure and the real issue-release adapter. Confirmed regression failures before the fix; regression tests, neighboring tests, and final typecheck pass. ## Verification - Reproducing test(s), added in this PR: `server/src/services/heartbeat/run-control-event-failure.test.ts`, `server/src/__tests__/heartbeat-lock-release-on-reassignment.test.ts`, `server/src/services/heartbeat/run-control.test.ts` - They fail on the base commit and pass with the fix, each twice in a fresh sandbox. - 3 independent attempts wrote 3 reproductions; this fix passes every one of them. - The repository's compile command passes with the fix applied. - 9 existing test file(s) near the change passed. - The full test suite passes except for the 16 file(s) that already fail on the base commit. ## Risks - Written by an automated pipeline, not a person: review it as you would an outside contribution. - Risk tier of the touched paths: 2. - It crosses a trust boundary. Check each crossing against what the issue asked for, and against who asked: - logging (server/src/services/heartbeat/run-control.ts:1665): Logs the event-append error and run ID to the server log when cancellation event recording fails. ## Model Used - codex:gpt-6.1-sol, opus, sonnet (Anthropic Claude, via Claude Code; OpenAI, via the Codex CLI), writing and checking the change in an isolated sandbox. - opus (Anthropic Claude, via Claude Code), reviewing it independently. ## Linked Issues Fixes #20116 The reviewer found the base already handles: - A closed request socket produces a misleading PATCH 200 log that hides the failure. (server/src/middleware/logger.ts:89–91 detects aborted or unfinished responses and formats 'request aborted'; lines 150 and 157–158 log these at warn level with that message instead of the default status-based success message.) A search of paperclipai/paperclip's open and closed pull requests found no other pull request that names #20116. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [ ] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with 'Fixes: #' / 'Closes #' / 'Refs #' OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub '#NNN' / 'github.com/paperclipai/paperclip' URLs) - [ ] My branch name describes the change (e.g. 'docs/...', 'fix/...') and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [ ] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge <!-- prospector:issue-fix v1 issue=20116 base=ca34bed25797 report=e2609f9803f7cc0c --> Co-authored-by: commitperclip[bot] <284812584+commitperclip[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
a6608bd735
commit
6f2ea3b4d5
4 files changed
+101
No files matched your search
@@ -36,6 +36,7 @@ vi.mock("@paperclipai/shared/telemetry", async () => {
|
||||
});
|
||||
|
||||
import { heartbeatService } from "../services/heartbeat.ts";
|
||||
import * as runEvents from "../services/heartbeat-run-events.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
@@ -159,6 +160,28 @@ describeEmbeddedPostgres("heartbeat lock release on cross-agent reassignment", (
|
||||
};
|
||||
}
|
||||
|
||||
it.each(["queued", "running"] as const)("releases a %s holder after the cancellation event append fails", async (holderStatus) => {
|
||||
const { coderAgentId, issueId, holderRunId, wakeupRequestId } =
|
||||
await seedCrossAgentScenario({ holderStatus });
|
||||
await db.update(agents).set({ status: "running" }).where(eq(agents.id, coderAgentId));
|
||||
const append = vi.spyOn(runEvents, "appendHeartbeatRunEvent").mockRejectedValueOnce(
|
||||
Object.assign(new Error("event append failed"), { code: "40P01" }),
|
||||
);
|
||||
try {
|
||||
const cancelled = await heartbeat.cancelRun(holderRunId);
|
||||
expect(cancelled?.status).toBe("cancelled");
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, holderRunId)))[0])
|
||||
.toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)))[0])
|
||||
.toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0])
|
||||
.toMatchObject({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null });
|
||||
expect((await db.select().from(agents).where(eq(agents.id, coderAgentId)))[0]?.status).toBe("idle");
|
||||
} finally {
|
||||
append.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("defers a cross-agent wake while the holder is still running and leaves the holder alone", async () => {
|
||||
const { coderAgentId, reviewerAgentId, issueId, holderRunId, wakeupRequestId } =
|
||||
await seedCrossAgentScenario({ holderStatus: "running" });
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { createHeartbeatRunControl } from "./run-control.js";
|
||||
import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js";
|
||||
import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js";
|
||||
import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js";
|
||||
|
||||
vi.mock("../local-service-supervisor.js", async (importOriginal) => ({
|
||||
...await importOriginal<typeof import("../local-service-supervisor.js")>(),
|
||||
terminateLocalService: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
describe.skipIf(!runControlPostgresSupport.supported)("cancellation event append failure", () => {
|
||||
let db: Db;
|
||||
const { fixture } = setupRunControlDatabase("paperclip-cancel-event-failure", value => { db = value; });
|
||||
beforeAll(async () => { await db.execute(sql`set client_min_messages to warning`); });
|
||||
it.each(["queued", "running"] as const)("finishes cancelling a %s run when its lifecycle event append fails", async status => {
|
||||
const { company, agent } = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id,
|
||||
source: "assignment", status: "claimed" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
|
||||
status, runtimeMode: "legacy", runtimeModeResolvedAt: new Date(), wakeupRequestId: wake.id }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Release after Stop",
|
||||
status: "in_progress", assigneeAgentId: agent.id, executionRunId: run.id,
|
||||
executionAgentNameKey: "controlled-agent", executionLockedAt: new Date() }).returning();
|
||||
await db.update(heartbeatRuns).set({ issueId: issue.id, contextSnapshot: { issueId: issue.id } })
|
||||
.where(eq(heartbeatRuns.id, run.id));
|
||||
const deps = callbacks(db);
|
||||
vi.mocked(deps.wakeQueue.releaseIssueExecution).mockImplementation(createReleaseIssueExecution({
|
||||
issueLock: createPostgresWakeQueueAdapter(db, {
|
||||
resolveResponsibleUserId: async () => "operator",
|
||||
getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }),
|
||||
resolveSessionBeforeForWakeup: async () => null,
|
||||
}),
|
||||
recovery: { escalateStrandedAssignedIssue: async () => undefined, escalateStrandedRecoveryIssueInPlace: async () => undefined },
|
||||
}));
|
||||
vi.mocked(deps.appendRunEvent).mockRejectedValueOnce(new Error("Synthetic event append failure"));
|
||||
|
||||
await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, undefined, {
|
||||
suppressImmediateRecovery: true,
|
||||
})).resolves.toMatchObject({ status: "cancelled" });
|
||||
|
||||
expect(await deps.getRun(run.id)).toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0])
|
||||
.toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, issue.id)))[0])
|
||||
.toMatchObject({ executionRunId: null, executionAgentNameKey: null, executionLockedAt: null });
|
||||
expect(deps.finalizeAgentStatus).toHaveBeenCalledExactlyOnceWith(agent.id, "cancelled", undefined, { wasFirstHeartbeat: undefined });
|
||||
expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledExactlyOnceWith(agent.id);
|
||||
});
|
||||
});
|
||||
@@ -184,6 +184,27 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat
|
||||
)).toMatchObject({ status: "cancelled" });
|
||||
});
|
||||
|
||||
it.each(["queued", "running", "scheduled_retry"] as const)("finishes cancellation settlement for %s when the event append fails", async status => {
|
||||
const { company, agent } = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({
|
||||
companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed",
|
||||
}).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({
|
||||
companyId: company.id, agentId: agent.id, status, wakeupRequestId: wake.id,
|
||||
runtimeMode: "legacy", runtimeModeResolvedAt: new Date(),
|
||||
}).returning();
|
||||
const deps = callbacks(db);
|
||||
vi.mocked(deps.appendRunEvent).mockRejectedValueOnce(new Error("event storage unavailable"));
|
||||
|
||||
expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" });
|
||||
expect(await deps.getRun(run.id)).toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0])
|
||||
.toMatchObject({ status: "cancelled", finishedAt: expect.any(Date) });
|
||||
expect(deps.wakeQueue.releaseIssueExecution).toHaveBeenCalledWith(expect.objectContaining({ runId: run.id, companyId: company.id }));
|
||||
expect(deps.finalizeAgentStatus).toHaveBeenCalledWith(agent.id, "cancelled", undefined, expect.any(Object));
|
||||
expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledExactlyOnceWith(agent.id);
|
||||
});
|
||||
|
||||
it.each([["40P01", 3], ["XX000", 1]] as const)("bounds cancellation transaction attempts for PostgreSQL %s", async (code, attempts) => {
|
||||
const { company, agent } = await fixture();
|
||||
const run = await liveRun(company.id, agent.id);
|
||||
|
||||
@@ -1676,6 +1676,11 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
level: "warn",
|
||||
message: options.eventMessage ?? "run cancelled",
|
||||
payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) },
|
||||
}).catch((eventErr) => {
|
||||
logger.warn(
|
||||
{ err: eventErr, runId: run.id },
|
||||
"failed to append run event for cancellation",
|
||||
);
|
||||
});
|
||||
await releaseIssueExecutionAndPromote(cancelled, {
|
||||
suppressImmediateRecovery: options.suppressImmediateRecovery,
|
||||
|
||||
Reference in new issue
Block a user