diff --git a/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts b/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts index 422f044097e..5ff2ecd93ef 100644 --- a/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts +++ b/server/src/__tests__/heartbeat-run-lease-release-terminalization.test.ts @@ -1,5 +1,5 @@ import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { costEvents, @@ -167,6 +167,96 @@ describeEmbeddedPostgres("heartbeat terminalizeRunOnLeaseRelease", () => { expect(terminal).toMatchObject({ status: "cancelled", error: null, errorCode: null }); }); + function personCancelRequest(input: { companyId: string; issueId: string; runId: string }) { + const requestedAt = new Date().toISOString(); + return { + cancellation: { reason: "Cancelled by a board operator", source: "operator", expected: true, + initiator: { id: "board-user", type: "user" }, recordedAt: requestedAt }, + cancelledByUserId: "board-user", + cancelledByActorType: "user", + executionCancellation: { state: "requested", requestedAt }, + startupCancellation: { errorCode: "cancelled", requestedAt, beforeNativeSelection: false }, + nativeCancellation: { + schema: "paperclip.native-cancellation.v1", runId: input.runId, companyId: input.companyId, issueId: input.issueId, + scope: "run", reasonCode: "cancellation_run_only", effects: ["release_run_resources"], dispatched: false, + dispatchState: "pending", intentId: `native-cancellation:${randomUUID()}`, intentAuditId: randomUUID(), + recordedAt: requestedAt, + }, + }; + } + + it("ends a run cancelled when a person's Stop was recorded before teardown read the run", async () => { + const { companyId, issueId, runId } = await seed({ issueStatus: "in_progress", runStatus: "running" }); + const [run] = await db.update(heartbeatRuns).set({ + nativeIssueId: issueId, + resultJson: personCancelRequest({ companyId, issueId, runId }), + }).where(eq(heartbeatRuns.id, runId)).returning(); + + const terminal = await heartbeatService(db).terminalizeRunOnLeaseRelease(run!); + + expect(terminal).toMatchObject({ status: "cancelled", errorCode: null }); + const row = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then((rows) => rows[0]); + expect(row?.status).toBe("cancelled"); + expect(row?.resultJson).toMatchObject({ cancelledByActorType: "user", cancelledByUserId: "board-user" }); + }); + + it("ends a run cancelled when a person's Stop lands after teardown read the run", async () => { + const { companyId, issueId, runId, run: staleRun } = await seed({ issueStatus: "in_progress", runStatus: "running" }); + await db.update(heartbeatRuns).set({ + nativeIssueId: issueId, + resultJson: personCancelRequest({ companyId, issueId, runId }), + }).where(eq(heartbeatRuns.id, runId)); + + const terminal = await heartbeatService(db).terminalizeRunOnLeaseRelease(staleRun); + + expect(terminal).toMatchObject({ status: "cancelled", errorCode: null }); + const row = await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }) + .from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then((rows) => rows[0]); + expect(row).toEqual({ status: "cancelled", errorCode: null }); + }); + + it("ends a run cancelled when a person's Stop commits while teardown's write waits", async () => { + const { companyId, issueId, runId, run: staleRun } = await seed({ issueStatus: "in_progress", runStatus: "running" }); + const other = createDb(tempDb!.connectionString); + let locked!: () => void, release!: () => void; + const lockHeld = new Promise(resolve => { locked = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + try { + const stop = other.transaction(async tx => { + await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update"); + locked(); + await gate; + await tx.update(heartbeatRuns).set({ nativeIssueId: issueId, resultJson: personCancelRequest({ companyId, issueId, runId }) }) + .where(eq(heartbeatRuns.id, runId)); + }); + await lockHeld; + const teardown = heartbeatService(db).terminalizeRunOnLeaseRelease(staleRun); + await vi.waitFor(async () => { + const waiting = await db.execute(sql`select 1 from pg_stat_activity + where wait_event_type = 'Lock' and datname = current_database() and query ilike '%heartbeat_runs%'`); + expect(waiting.length).toBeGreaterThan(0); + }, { timeout: 10_000 }); + release(); + await stop; + expect(await teardown).toMatchObject({ status: "cancelled", errorCode: null }); + } finally { + release(); + await other.$client.end(); + } + }); + + it("still reports interruption when only an agent requested the stop", async () => { + const { companyId, issueId, runId } = await seed({ issueStatus: "in_progress", runStatus: "running" }); + const request = personCancelRequest({ companyId, issueId, runId }); + const [run] = await db.update(heartbeatRuns).set({ + resultJson: { ...request, cancelledByActorType: "agent", cancelledByUserId: undefined }, + }).where(eq(heartbeatRuns.id, runId)).returning(); + + const terminal = await heartbeatService(db).terminalizeRunOnLeaseRelease(run!); + + expect(terminal).toMatchObject({ status: "interrupted", errorCode: "lease_released_before_terminal" }); + }); + it("preserves an acknowledged agent termination Stop when teardown wins the finalizer race", async () => { const { companyId, issueId, runId } = await seed({ issueStatus: "in_progress", runStatus: "running" }); const [run] = await db.update(heartbeatRuns).set({ diff --git a/server/src/services/acknowledged-native-stop.ts b/server/src/services/acknowledged-native-stop.ts index 25beefee409..b6be592360e 100644 --- a/server/src/services/acknowledged-native-stop.ts +++ b/server/src/services/acknowledged-native-stop.ts @@ -1,4 +1,4 @@ -import { and, desc, eq, inArray, isNotNull } from "drizzle-orm"; +import { and, desc, eq, inArray, isNotNull, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; import { parseObject } from "../adapters/utils.js"; import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; @@ -34,6 +34,23 @@ export function hasAcknowledgedNativeStopIntent(run: Parameters | null }): boolean { + const result = run.resultJson; + if (result?.cancelledByActorType !== "user" || typeof result.cancelledByUserId !== "string" || !result.cancelledByUserId) return false; + const state = parseObject(result.executionCancellation).state; + const startup = result.startupCancellation; + return state === "requested" || state === "acknowledged" || + (Boolean(startup) && typeof startup === "object" && !Array.isArray(startup)); +} + +export function recordedUserStopRequestSql() { + const result = heartbeatRuns.resultJson; + return sql`(coalesce(${result}->>'cancelledByActorType', '') = 'user' + and coalesce(${result}->>'cancelledByUserId', '') <> '' + and (coalesce(${result}->'executionCancellation'->>'state', '') in ('requested', 'acknowledged') + or jsonb_typeof(${result}->'startupCancellation') = 'object'))`; +} + export function hasAcknowledgedNativeLifecycleStopIntent(run: Parameters[0]): boolean { return isAgentLifecycleStopRecord(run.resultJson) && hasAcknowledgedNativeRunStopReceipt(run); } diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index f4837170077..77e1030f039 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -50,6 +50,8 @@ import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeLifecycleStopIntent, hasAcknowledgedNativeStopIntent, + hasRecordedUserStopRequest, + recordedUserStopRequestSql, } from "../acknowledged-native-stop.js"; import { connectionIntentService } from "../connection-intents.js"; import { @@ -1376,54 +1378,70 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl // released. It never overwrites a status that another path already made // terminal. async function terminalizeRunOnLeaseRelease( - run: typeof heartbeatRuns.$inferSelect, + staleRun: typeof heartbeatRuns.$inferSelect, ): Promise { - if (isNativeRunnerOwnershipHeld(run)) return run; - if (isHeartbeatRunTerminalStatus(run.status)) return run; - if (run.status !== "running" && run.status !== "queued") return run; + if (isNativeRunnerOwnershipHeld(staleRun)) return staleRun; + if (isHeartbeatRunTerminalStatus(staleRun.status)) return staleRun; + if (staleRun.status !== "running" && staleRun.status !== "queued") return staleRun; + let run = staleRun; + let write: Awaited> | null = null; + let terminalStatus: "succeeded" | "cancelled" | "interrupted" = "interrupted"; + let issueId: string | null = null; + for (let attempt = 0; attempt < 2; attempt++) { + const current = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)) + .then((rows) => rows[0] ?? null); + if (!current) return run; + run = current; + if (isNativeRunnerOwnershipHeld(run)) return run; + if (isHeartbeatRunTerminalStatus(run.status)) return run; + if (run.status !== "running" && run.status !== "queued") return run; - // Choose the terminal status that reflects the true outcome. When the issue - // already reached a terminal status, the run reached its goal, so use the - // matching terminal run status. Otherwise the teardown cut the run short, - // so use "interrupted". - const issueId = readNonEmptyString( - parseObject(run.contextSnapshot).issueId, - ); - let terminalStatus: "succeeded" | "cancelled" | "interrupted" = - "interrupted"; - if (issueId) { - const issueStatus = await db - .select({ status: issues.status }) - .from(issues) - .where(eq(issues.id, issueId)) - .then((rows) => rows[0]?.status ?? null); - if (issueStatus === "done") terminalStatus = "succeeded"; - else if (issueStatus === "cancelled") terminalStatus = "cancelled"; + // Choose the terminal status that reflects the true outcome. When the issue + // already reached a terminal status, the run reached its goal, so use the + // matching terminal run status. Otherwise the teardown cut the run short, + // so use "interrupted". + issueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); + terminalStatus = "interrupted"; + if (issueId) { + const issueStatus = await db + .select({ status: issues.status }) + .from(issues) + .where(eq(issues.id, issueId)) + .then((rows) => rows[0]?.status ?? null); + if (issueStatus === "done") terminalStatus = "succeeded"; + else if (issueStatus === "cancelled") terminalStatus = "cancelled"; + } + + // Teardown can beat the cancellation finalizer. Preserve the acknowledged + // user intent instead of reporting an infrastructure interruption. + if (hasRecordedUserStopRequest(run) || hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run) + || hasAcknowledgedNativeLifecycleStopIntent(run)) terminalStatus = "cancelled"; + + // Match both "running" and "queued". A queued run has released its lease but + // never reached "running", so a running-only update would miss it and leave + // a phantom live run behind. + write = await setRunStatusFromLive( + run.id, + terminalStatus, + ["running", "queued"], + { + finishedAt: run.finishedAt ?? new Date(), + error: run.error ?? (terminalStatus === "interrupted" + ? `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown` : null), + errorCode: + run.errorCode ?? + (terminalStatus === "interrupted" + ? "lease_released_before_terminal" + : null), + }, + undefined, + terminalStatus === "cancelled" ? undefined : not(recordedUserStopRequestSql()), + ); + if (write.updated) break; + if (!write.run || isHeartbeatRunTerminalStatus(write.run.status)) break; } - - // Teardown can beat the cancellation finalizer. Preserve the acknowledged - // user intent instead of reporting an infrastructure interruption. - if (hasAcknowledgedNativeStopIntent(run) || hasAcknowledgedNativeReassignmentStopIntent(run) - || hasAcknowledgedNativeLifecycleStopIntent(run)) terminalStatus = "cancelled"; - + if (!write) return run; const message = `run terminalized on environment lease release: heartbeat_runs.status was still ${run.status} at teardown`; - // Match both "running" and "queued". A queued run has released its lease but - // never reached "running", so a running-only update would miss it and leave - // a phantom live run behind. - const write = await setRunStatusFromLive( - run.id, - terminalStatus, - ["running", "queued"], - { - finishedAt: run.finishedAt ?? new Date(), - error: run.error ?? (terminalStatus === "interrupted" ? message : null), - errorCode: - run.errorCode ?? - (terminalStatus === "interrupted" - ? "lease_released_before_terminal" - : null), - }, - ); if (!write.updated) { // Another path already finalized the run. Keep that terminal outcome. return write.run ?? run;