diff --git a/doc/plans/2026-11-13-thousand-concurrent-agents.md b/doc/plans/2026-11-13-thousand-concurrent-agents.md index 58e0164dea8..c9c447a53e4 100644 --- a/doc/plans/2026-11-13-thousand-concurrent-agents.md +++ b/doc/plans/2026-11-13-thousand-concurrent-agents.md @@ -326,6 +326,16 @@ The master row has much less run churn than the earlier rows, so compare the per-window totals with care. In that window the one backstop pass loaded 260 tasks into the stranded queue; run events loaded 147. +A graceful shutdown now releases the process identity of the settled native +runs that the stopping server ran, once it sees that their local processes +exited. It uses one candidate read and one update for all runs (7 statements +for 40 runs). Before this change, those runs kept `process_pid` and +`process_group_id` after the deploy. A successor in another container cannot +release them. A successor on the same host releases them at startup at about +10 statements per run (401 statements for 40 runs); after a graceful shutdown +it does one statement. Runs still running at shutdown, sandbox runs, and runs +of other servers keep their records. + ### Phase 2: several controller processes (target: 1,000) | # | Change | diff --git a/server/src/__tests__/shutdown-settled-process-release.test.ts b/server/src/__tests__/shutdown-settled-process-release.test.ts new file mode 100644 index 00000000000..720658479b0 --- /dev/null +++ b/server/src/__tests__/shutdown-settled-process-release.test.ts @@ -0,0 +1,221 @@ +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, applyPendingMigrations, authUsers, closeCheckpointDb, closeLivenessDb, companies, companyMemberships, + createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-lifecycle.js"; +import { currentProcessSpaceHash } from "../services/controller-process-identity.js"; +import { + discoverSettledNativeRunProcesses, + hasNativeLocalProcessStop, + releaseOwnedSettledNativeRunProcessesAtShutdown, + withReleasedNativeRunProcess, +} from "../services/native-local-process-stop.js"; + +vi.setConfig({ testTimeout: 300_000, hookTimeout: 180_000 }); + +const SETTLED = ["failed", "cancelled", "timed_out", "interrupted"] as const; + +describe("graceful shutdown releases the process identity of settled native runs", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let meter: ReturnType; + const children: number[] = []; + + beforeAll(async () => { + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_SHUTDOWN_PROCESS_RELEASE_TEST_DATABASE_URL"); + if (!external) temporary = await startEmbeddedPostgresTestDatabase("shutdown-process-release-"); + else await applyPendingMigrations(external); + const url = external ?? temporary!.connectionString; + db = createDb(url, { maxConnections: 2, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + meter = meterDatabaseStatements(db, url); + }); + beforeEach(async () => { + await db.execute(sql`truncate companies, controller_instances cascade`); + }); + afterEach(() => { + for (const pid of children.splice(0)) { try { process.kill(-pid, "SIGKILL"); } catch { } } + }); + afterAll(async () => { + meter?.uninstall(); + if (db) { await closeLivenessDb(db, 1); await closeCheckpointDb(db, 1); } + await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); + }); + + async function fixture() { + const companyId = randomUUID(), actorId = "shutdown-release-user"; + await db.insert(authUsers).values({ id: actorId, name: "Owner", email: `${actorId}@example.test`, + createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companies).values({ id: companyId, name: "Shutdown", issuePrefix: `SD${companyId.slice(0, 6)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: actorId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, + status: "active", membershipRole: "owner" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Worker", status: "idle", adapterType: "process", + adapterConfig: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Work", status: "done", assigneeAgentId: agent.id }).returning(); + return { companyId, agentId: agent.id, issueId: issue.id }; + } + type Fixture = Awaited>; + + async function exitedPid() { + const child = spawn(process.execPath, ["-e", "0"], { detached: true, stdio: "ignore" }); + const pid = child.pid!; await once(child, "exit"); return pid; + } + function livePid() { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + child.unref(); children.push(child.pid!); return child.pid!; + } + + type Seed = { pid: number; status?: string; foreign?: boolean; remote?: boolean; phase?: string; + leasePending?: boolean }; + async function seed(f: Fixture, options: Seed) { + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, runtimeMode: "native", + nativeIssueId: f.issueId, status: "running", invocationSource: "assignment", triggerDetail: "system", + runnerProfileJson: { sessionCheckpoint: { providerSessionId: "thread" } } }).returning(); + const startedAt = new Date(Date.now() - 60_000); + if (options.foreign || options.remote) { + await appendHeartbeatRunEvent(db, { companyId: f.companyId, runId: run.id, agentId: f.agentId, + eventType: "native.process_identity_recorded", stream: "system", level: "info", message: "identity", + payload: { targetKind: options.remote ? "remote" : "local", hostname: "replaced-container", + processPid: options.pid, processGroupId: options.pid, processStartedAt: startedAt.toISOString() }, + runUpdate: { processPid: options.pid, processGroupId: options.pid, processStartedAt: startedAt, + processHostHash: options.remote ? "remote" : "replaced-container-space" } }); + } else { + await persistHeartbeatRunProcessMetadata(db, run.id, { pid: options.pid, processGroupId: options.pid, + startedAt: startedAt.toISOString(), targetKind: "local" }); + } + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId, runId: run.id, + phase: options.phase ?? (options.status === "running" ? "observed" : "committed"), controllerBootId: randomUUID() }); + await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: run.id, + provider: options.remote ? "daytona" : "local", providerLeaseId: options.remote ? randomUUID() : null, + leasePolicy: "ephemeral", status: options.leasePending ? "active" : "released", + releasedAt: options.leasePending ? null : new Date(), cleanupStatus: options.leasePending ? null : "success" }); + if (options.status !== "running") { + await db.update(heartbeatRuns).set({ status: options.status ?? "failed", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, run.id)); + } + return run.id; + } + + const strandedSettledRuns = async (companyId: string) => (await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.runtimeMode, "native"), + inArray(heartbeatRuns.status, [...SETTLED]), sql`${heartbeatRuns.finishedAt} is not null`, + sql`(${heartbeatRuns.processPid} is not null or ${heartbeatRuns.processGroupId} is not null)`))).map(row => row.id).sort(); + const processFields = async (id: string) => (await db.select({ pid: heartbeatRuns.processPid, group: heartbeatRuns.processGroupId, + startedAt: heartbeatRuns.processStartedAt, hostHash: heartbeatRuns.processHostHash }) + .from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + const stopReceipts = async (id: string) => db.select().from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.runId, id), eq(heartbeatRunEvents.eventType, "native.local_process_stopped"))); + + async function seedDeploy(f: Fixture, owned: number) { + const exited = await exitedPid(); + const ownedRuns: string[] = []; + for (let index = 0; index < owned; index++) { + ownedRuns.push(await seed(f, { pid: exited, status: SETTLED[index % SETTLED.length] })); + } + const kept = { + suspendedRunning: await seed(f, { pid: livePid(), status: "running" }), + settledButAlive: await seed(f, { pid: livePid() }), + remoteSandbox: await seed(f, { pid: 77, remote: true }), + otherContainer: await seed(f, { pid: exited, foreign: true }), + coordinatorUnsettled: await seed(f, { pid: exited, phase: "workspace_finalizing" }), + leaseCleanupPending: await seed(f, { pid: exited, leasePending: true }), + }; + return { exited, ownedRuns, kept }; + } + + it("leaves no stranded identity on the settled runs it ran, in a fixed number of statements, and touches nothing else", async () => { + const f = await fixture(); + const { exited, ownedRuns, kept } = await seedDeploy(f, 40); + const keptBefore = Object.fromEntries(await Promise.all(Object.entries(kept) + .map(async ([name, id]) => [name, await processFields(id)] as const))); + expect(await strandedSettledRuns(f.companyId)).toHaveLength(ownedRuns.length + 5); + + const before = meter.read(); + const release = await releaseOwnedSettledNativeRunProcessesAtShutdown(db); + const statements = meter.read().statements - before.statements; + + expect(release).toEqual({ candidates: ownedRuns.length + 1, stillRunning: 1, released: ownedRuns.length }); + expect(statements).toBeLessThanOrEqual(7); + expect(await strandedSettledRuns(f.companyId)).toEqual([kept.settledButAlive, kept.remoteSandbox, kept.otherContainer, + kept.coordinatorUnsettled, kept.leaseCleanupPending].sort()); + for (const [name, id] of Object.entries(kept)) expect(await processFields(id)).toEqual(keptBefore[name]); + for (const [name, id] of Object.entries(kept)) expect({ name, receipts: (await stopReceipts(id)).length }).toEqual({ name, receipts: 0 }); + + for (const runId of ownedRuns) { + const fields = await processFields(runId); + expect(fields).toMatchObject({ pid: null, group: null, startedAt: null, hostHash: currentProcessSpaceHash() }); + const receipts = await stopReceipts(runId); + expect(receipts).toHaveLength(1); + expect(receipts[0]!.payload).toMatchObject({ processPid: exited, processGroupId: exited }); + expect(await hasNativeLocalProcessStop(db, f.companyId, runId)).toBe(true); + const restored = await withReleasedNativeRunProcess(db, { id: runId, companyId: f.companyId, + processPid: null, processGroupId: null, processStartedAt: null }); + expect(restored).toMatchObject({ processPid: exited, processGroupId: exited }); + } + + const again = await releaseOwnedSettledNativeRunProcessesAtShutdown(db); + expect(again).toEqual({ candidates: 1, stillRunning: 1, released: 0 }); + }); + + it("leaves the next server on this host nothing to release, where an abrupt stop leaves it every run", async () => { + const graceful = await fixture(); + const gracefulDeploy = await seedDeploy(graceful, 40); + await releaseOwnedSettledNativeRunProcessesAtShutdown(db); + const skipKept = (id: string) => Object.values(gracefulDeploy.kept).includes(id); + let before = meter.read(); + const afterGraceful = await discoverSettledNativeRunProcesses(db, { skip: skipKept }); + const gracefulStatements = meter.read().statements - before.statements; + + await db.execute(sql`truncate companies, controller_instances cascade`); + const abrupt = await fixture(); + const abruptDeploy = await seedDeploy(abrupt, 40); + const strandedAfterAbruptStop = await strandedSettledRuns(abrupt.companyId); + before = meter.read(); + const afterAbrupt = await discoverSettledNativeRunProcesses(db, + { skip: id => Object.values(abruptDeploy.kept).includes(id) }); + const abruptStatements = meter.read().statements - before.statements; + + expect(afterGraceful.released).toBe(0); + expect(gracefulStatements).toBe(1); + expect(strandedAfterAbruptStop).toHaveLength(abruptDeploy.ownedRuns.length + 5); + expect(afterAbrupt.released).toBe(abruptDeploy.ownedRuns.length); + expect(abruptStatements).toBeGreaterThan(abruptDeploy.ownedRuns.length * 5); + }); + + it("releases nothing when the server is killed before its shutdown step, so a successor in another container sees today's rows", async () => { + const f = await fixture(); + const { ownedRuns } = await seedDeploy(f, 12); + const snapshot = await Promise.all(ownedRuns.map(processFields)); + expect(await strandedSettledRuns(f.companyId)).toHaveLength(ownedRuns.length + 5); + expect(await Promise.all(ownedRuns.map(processFields))).toEqual(snapshot); + for (const runId of ownedRuns) expect(await stopReceipts(runId)).toHaveLength(0); + }); + + it("never releases a run whose recorded identity changed after it was read", async () => { + const f = await fixture(); + const exited = await exitedPid(); + const runId = await seed(f, { pid: exited }); + const replacement = livePid(); + const transaction = db.transaction.bind(db); + const spy = vi.spyOn(db, "transaction").mockImplementationOnce((async (run: Parameters[0]) => { + await db.update(heartbeatRuns).set({ processPid: replacement, processGroupId: replacement }).where(eq(heartbeatRuns.id, runId)); + return transaction(run); + }) as typeof db.transaction); + try { + const release = await releaseOwnedSettledNativeRunProcessesAtShutdown(db); + expect(spy).toHaveBeenCalledTimes(1); + expect(release).toEqual({ candidates: 1, stillRunning: 0, released: 0 }); + } finally { + spy.mockRestore(); + } + expect(await processFields(runId)).toMatchObject({ pid: replacement, group: replacement }); + expect(await stopReceipts(runId)).toHaveLength(0); + }); +}); diff --git a/server/src/index.ts b/server/src/index.ts index 340b576632c..0cb0eb45c61 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -224,6 +224,10 @@ import { loadWithoutCoordinatedShutdownSignalHooks, runBoundedShutdownStep, } from "./shutdown.js"; +import { + releaseOwnedSettledNativeRunProcessesAtShutdown, + type ShutdownSettledProcessRelease, +} from "./services/native-local-process-stop.js"; import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js"; import { initializeAdapterSettings } from "./services/adapter-settings.js"; import { migrateCloudConfiguration } from "./services/cloud-configuration-migration.js"; @@ -1303,6 +1307,7 @@ async function startServerWithDatabaseTeardown( ) => Promise) | null = null; let drainHeartbeatExecutionFinalizers: (() => Promise) | null = null; let handOffNativeRunsForShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise) | null = null; + let releaseSettledNativeProcessesForShutdown: (() => Promise) | null = null; let prepareHotRestartShutdown: ((signal: "SIGINT" | "SIGTERM", now?: Date, abort?: AbortSignal) => Promise<{ skipDrain: boolean; drainRunIds?: string[]; @@ -1644,6 +1649,7 @@ async function startServerWithDatabaseTeardown( drainHeartbeatExecutionFinalizers = () => heartbeat.drainActiveRunExecutions(); handOffNativeRunsForShutdown = signal => heartbeat.handOffNativeRunsForShutdown(signal); + releaseSettledNativeProcessesForShutdown = () => releaseOwnedSettledNativeRunProcessesAtShutdown(db as any); prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown; const routines = routineService(db as any, { pluginWorkerManager }); const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat); @@ -2336,6 +2342,21 @@ async function startServerWithDatabaseTeardown( await workspaceRecoveryStop; + const releaseSettledNativeProcesses = releaseSettledNativeProcessesForShutdown; + if (releaseSettledNativeProcesses) { + await runBoundedShutdownStep({ + step: "settled_native_process_release", + signal, + timeoutMs: 3_000, + run: async () => { + const release = await releaseSettledNativeProcesses(); + logger.info({ signal, ...release }, "released the process identity of settled native runs this server ran"); + return release; + }, + log: logger, + }); + } + const telemetryClient = getTelemetryClient(); if (telemetryClient) { telemetryClient.stop(); diff --git a/server/src/services/instruction-collection-notifications.ts b/server/src/services/instruction-collection-notifications.ts index 74c607f8359..34fefeec033 100644 --- a/server/src/services/instruction-collection-notifications.ts +++ b/server/src/services/instruction-collection-notifications.ts @@ -11,7 +11,7 @@ export async function notifyInstructionCollection(transaction: object, scope?: I await notifyDeliveryWork(transaction, DELIVERY_QUEUES.instructionCollection, scope ? JSON.stringify(scope) : "*"); } -const waitingCopy = (scope: { companyId: string; runId: string }) => sql`company_id = ${scope.companyId} and run_id = ${scope.runId} and ( +const waitingCopyState = sql`( (not (coalesce(receipt, '{}'::jsonb) ? 'retainedByRunId') and ( (attempts < 3 and state in ('prepared', 'pending_collection', 'unchanged_turn', 'warm_saved')) or (state = 'unavailable' and process_stopped_at is null and location like 'remote:%' @@ -20,6 +20,9 @@ const waitingCopy = (scope: { companyId: string; runId: string }) => sql`company and receipt->>'schema' = 'paperclip.agent-files.v1' and coalesce(receipt->>'retainedByRunId', '') = '' and (receipt->>'cleanupPending' = 'true' or (receipt->>'cleanupPending' is distinct from 'false' and receipt ? 'baseline'))))`; +const waitingCopy = (scope: { companyId: string; runId: string }) => + sql`company_id = ${scope.companyId} and run_id = ${scope.runId} and ${waitingCopyState}`; + export async function notifyInstructionCopyStopEvidence(transaction: object, scope: { companyId: string; runId: string }) { if (typeof (transaction as Db).execute !== "function") return; const found = await (transaction as Db).execute(sql`select 1 from agent_instruction_working_copies where ${waitingCopy(scope)}`); @@ -30,3 +33,17 @@ export async function notifyInstructionCopyStopEvidence(transaction: object, sco set next_attempt_at = least(coalesce(next_attempt_at, now()), now()), updated_at = clock_timestamp() where ${waitingCopy(scope)}`); } + +export async function notifyInstructionCopyStopEvidenceForRuns(transaction: object, + scopes: ReadonlyArray<{ companyId: string; runId: string }>) { + if (scopes.length === 0 || typeof (transaction as Db).execute !== "function") return; + for (const scope of scopes) { + await notifyInstructionCollection(transaction, scope); + await notifyDirectoryCleanup(transaction, scope); + } + const runs = JSON.stringify(scopes.map(scope => ({ company_id: scope.companyId, run_id: scope.runId }))); + await (transaction as Db).execute(sql`update agent_instruction_working_copies + set next_attempt_at = least(coalesce(next_attempt_at, now()), now()), updated_at = clock_timestamp() + where (company_id, run_id) in (select company_id, run_id from jsonb_to_recordset(${runs}::jsonb) as stopped(company_id uuid, run_id uuid)) + and ${waitingCopyState}`); +} diff --git a/server/src/services/native-local-process-stop.ts b/server/src/services/native-local-process-stop.ts index 8b1b7c867bc..527d18063aa 100644 --- a/server/src/services/native-local-process-stop.ts +++ b/server/src/services/native-local-process-stop.ts @@ -1,5 +1,6 @@ import { notifyLiveRunRecovery } from "./live-run-recovery-notifications.js"; -import { notifyInstructionCopyStopEvidence } from "./instruction-collection-notifications.js"; +import { notifyInstructionCollection, notifyInstructionCopyStopEvidence, notifyInstructionCopyStopEvidenceForRuns } from "./instruction-collection-notifications.js"; +import { notifyDirectoryCleanup } from "./directory-cleanup-notifications.js"; import { and, desc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; import { notifyRetainedNativeSessionCleanup } from "./retained-native-session-cleanup-notifications.js"; @@ -543,3 +544,91 @@ export async function discoverSettledNativeRunProcesses(db: Db, input: { } while (after); return { released }; } + +export const SHUTDOWN_SETTLED_PROCESS_RELEASE_LIMIT = 500; +const SHUTDOWN_SETTLED_PROCESS_STATEMENT_TIMEOUT_MS = 2_000; + +const isoTimestamp = (column: string) => + sql.raw(`to_char(${column} at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`); + +function ownedSettledLocalProcessPredicate(processSpace: string) { + return sql`r.runtime_mode = 'native' and r.status in ('failed', 'cancelled', 'timed_out', 'interrupted') + and r.finished_at is not null and r.process_host_hash = ${processSpace} + and (r.process_pid is not null or r.process_group_id is not null) + and not exists (select 1 from native_run_finalizations f where f.run_id = r.id + and f.phase not in ('committed', 'terminal_failure')) + and (coalesce(r.runner_profile_json->'sessionCheckpoint', 'null'::jsonb) <> 'null'::jsonb + or exists (select 1 from heartbeat_run_events e where e.company_id = r.company_id and e.run_id = r.id + and e.event_type in (${providerEventTypes()}))) + and not exists (select 1 from environment_leases l where l.company_id = r.company_id and l.heartbeat_run_id = r.id + and (l.provider is distinct from 'local' or l.released_at is null or l.status = 'pending_cleanup' or l.cleanup_status = 'failed')) + and coalesce((select identity.payload->>'targetKind' = 'local' and identity.payload->>'hostname' = ${os.hostname()}::text + and identity.payload ? 'processPid' and identity.payload->'processPid' = coalesce(to_jsonb(r.process_pid), 'null'::jsonb) + and identity.payload ? 'processGroupId' and identity.payload->'processGroupId' = coalesce(to_jsonb(r.process_group_id), 'null'::jsonb) + and case when r.process_started_at is not null + then identity.payload->>'processStartedAt' = ${isoTimestamp("r.process_started_at")} + else coalesce(r.process_pid > 1 and r.process_group_id = r.process_pid, false) end + from (select e.payload from heartbeat_run_events e where e.company_id = r.company_id and e.run_id = r.id + and e.event_type = ${PROCESS_IDENTITY_RECORDED}::text and e.source_event_id is null order by e.seq desc limit 1) identity + ), false)`; +} + +type OwnedSettledProcess = { id: string; companyId: string; agentId: string; processPid: number | null; + processGroupId: number | null; processStartedAt: string | null; processStartedAtIso: string | null }; + +export type ShutdownSettledProcessRelease = { candidates: number; stillRunning: number; released: number }; + +export async function releaseOwnedSettledNativeRunProcessesAtShutdown(db: Db, options: { limit?: number } = {}): + Promise { + const processSpace = currentProcessSpaceHash(); + if (processSpace === UNKNOWN_PROCESS_SPACE) return { candidates: 0, stillRunning: 0, released: 0 }; + const limit = Math.max(1, Math.min(options.limit ?? SHUTDOWN_SETTLED_PROCESS_RELEASE_LIMIT, SHUTDOWN_SETTLED_PROCESS_RELEASE_LIMIT)); + const candidates = await db.execute(sql`select r.id, r.company_id as "companyId", r.agent_id as "agentId", + r.process_pid as "processPid", r.process_group_id as "processGroupId", r.process_started_at::text as "processStartedAt", + ${isoTimestamp("r.process_started_at")} as "processStartedAtIso" + from heartbeat_runs r where ${ownedSettledLocalProcessPredicate(processSpace)} + order by r.id limit ${limit}`) as unknown as OwnedSettledProcess[]; + const stopped = candidates.filter(run => (!run.processPid || absent(run.processPid)) + && (!run.processGroupId || absent(-run.processGroupId))); + if (stopped.length === 0) return { candidates: candidates.length, stillRunning: candidates.length, released: 0 }; + const recorded = JSON.stringify(stopped.map(run => ({ id: run.id, pid: run.processPid, pgid: run.processGroupId, + started: run.processStartedAt, started_iso: run.processStartedAtIso }))); + const released = await db.transaction(async transaction => { + const tx = transaction as unknown as Db; + await tx.execute(sql.raw(`set local statement_timeout = ${SHUTDOWN_SETTLED_PROCESS_STATEMENT_TIMEOUT_MS}`)); + for (const run of stopped) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionReplacement, JSON.stringify({ companyId: run.companyId, runId: run.id })); + await notifyLiveRunRecovery(tx, { companyId: run.companyId, runId: run.id, agentId: run.agentId }); + await notifyRetainedNativeSessionCleanup(tx, { companyId: run.companyId, runId: run.id }); + await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id }); + await notifyDirectoryCleanup(tx, { companyId: run.companyId, runId: run.id }); + } + for (const companyId of new Set(stopped.map(run => run.companyId))) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); + } + const rows = await tx.execute(sql`with stopped as ( + select * from jsonb_to_recordset(${recorded}::jsonb) as stopped(id uuid, pid integer, pgid integer, started text, started_iso text) + ), released as ( + update heartbeat_runs r set process_pid = null, process_group_id = null, process_started_at = null, + next_event_seq = r.next_event_seq + 1, updated_at = now() + from stopped + where r.id = stopped.id and r.process_pid is not distinct from stopped.pid + and r.process_group_id is not distinct from stopped.pgid + and r.process_started_at is not distinct from stopped.started::timestamptz + and ${ownedSettledLocalProcessPredicate(processSpace)} + returning r.company_id, r.id as run_id, r.agent_id, r.next_event_seq - 1 as seq, + stopped.pid, stopped.pgid, stopped.started_iso + ) + insert into heartbeat_run_events (company_id, run_id, agent_id, seq, event_type, stream, level, message, payload) + select company_id, run_id, agent_id, seq, ${LOCAL_PROCESS_STOPPED}, 'system', 'info', + 'The server that ran this local process verified it stopped before shutting down and cleared its identity.', + jsonb_build_object('processPid', pid, 'processGroupId', pgid, 'processStartedAt', started_iso) + from released + returning company_id as "companyId", run_id as "runId", agent_id as "agentId"`) as unknown as + Array<{ companyId: string; runId: string; agentId: string }>; + if (rows.length > 0) await notifyInstructionCopyStopEvidenceForRuns(tx, rows); + return rows; + }); + for (const run of released) pendingSettledProcessReleases.delete(run.runId); + return { candidates: candidates.length, stillRunning: candidates.length - stopped.length, released: released.length }; +}