diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 99f28b85a3b..197b10cb495 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -507,6 +507,16 @@ indexes also serve the comment and delivery `on delete set null` actions. Issue lists share one external review-wait read per issue between the review and blocked-inbox attention maps. +Migration 0487 builds two partial `heartbeat_runs` indexes concurrently for the +settled-process recovery queue. `heartbeat_runs_settled_process_identity_idx` +keys settled native runs that still hold a process identity by +`process_host_hash`, company and id. Each server reads only the runs recorded in +its own PID space, sandbox runs (`remote`) and rows saved before the hash, so +finished runs from replaced containers are not read at all. +`heartbeat_runs_pending_api_stop_idx` keys cancelled `cursor_cloud_api` runs +whose Stop is still `requested`. Without them, the queue reads every finished +run of a company on each hint. + Migration 0476 builds `agent_wakeup_requests_company_run_idx` concurrently. It keys wake requests by company and `run_id`, partial on a non-null run. Queued-run dispatch finds the person wakes that joined or were adopted by each queued run diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 2860e95c383..4145a5ecb1d 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -748,6 +748,7 @@ const MIGRATIONS_WITH_TOP_LEVEL_STATEMENTS = new Set([ "0472_runner_api_capture_pending_deletes.sql", "0476_wakeup_request_run_index.sql", "0486_heartbeat_runs_company_finished_index.sql", + "0487_heartbeat_runs_settled_process_indexes.sql", "0469_zippy_photon.sql", diff --git a/packages/db/src/migrations/0487_heartbeat_runs_settled_process_indexes.sql b/packages/db/src/migrations/0487_heartbeat_runs_settled_process_indexes.sql new file mode 100644 index 00000000000..1e1cd989a68 --- /dev/null +++ b/packages/db/src/migrations/0487_heartbeat_runs_settled_process_indexes.sql @@ -0,0 +1,5 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "heartbeat_runs_settled_process_identity_idx" ON "heartbeat_runs" USING btree ("process_host_hash","company_id","id") WHERE "heartbeat_runs"."runtime_mode" = 'native' and "heartbeat_runs"."status" in ('failed', 'cancelled', 'timed_out', 'interrupted') + and "heartbeat_runs"."finished_at" is not null + and ("heartbeat_runs"."process_pid" is not null or "heartbeat_runs"."process_group_id" is not null or "heartbeat_runs"."process_started_at" is not null);--> statement-breakpoint +CREATE INDEX CONCURRENTLY IF NOT EXISTS "heartbeat_runs_pending_api_stop_idx" ON "heartbeat_runs" USING btree ("company_id","id") WHERE "heartbeat_runs"."status" = 'cancelled' and "heartbeat_runs"."driver_kind" = 'cursor_cloud_api' + and ("heartbeat_runs"."result_json"->'executionCancellation'->>'state') = 'requested'; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0482_snapshot.json b/packages/db/src/migrations/meta/0487_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0482_snapshot.json rename to packages/db/src/migrations/meta/0487_snapshot.json index fe3c626a03e..c9758d85553 100644 --- a/packages/db/src/migrations/meta/0482_snapshot.json +++ b/packages/db/src/migrations/meta/0487_snapshot.json @@ -1,6 +1,6 @@ { - "id": "8b353a38-62df-4834-be55-7e9ebefad5c5", - "prevId": "41eacc6a-6eab-4f01-965a-62aa0cd6778f", + "id": "73c8fa97-70fc-49c0-804e-e49cfdffd156", + "prevId": "0c3e74be-7936-4c97-9798-505daf53c16b", "version": "7", "dialect": "postgresql", "tables": { @@ -6582,6 +6582,111 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.automatic_run_brakes": { + "name": "automatic_run_brakes", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'closed'" + }, + "reason": { + "name": "reason", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "state_changed_at": { + "name": "state_changed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_probe_at": { + "name": "next_probe_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "probe_run_id": { + "name": "probe_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "probe_successes": { + "name": "probe_successes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "release_window_started_at": { + "name": "release_window_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_window_starts": { + "name": "release_window_starts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_creation_at": { + "name": "next_creation_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automatic_run_brakes_company_id_companies_id_fk": { + "name": "automatic_run_brakes_company_id_companies_id_fk", + "tableFrom": "automatic_run_brakes", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.board_api_keys": { "name": "board_api_keys", "schema": "", @@ -36029,6 +36134,78 @@ "method": "btree", "with": {} }, + "heartbeat_runs_company_finished_idx": { + "name": "heartbeat_runs_company_finished_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finished_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"finished_at\" is not null", + "concurrently": true, + "method": "btree", + "with": {} + }, + "heartbeat_runs_settled_process_identity_idx": { + "name": "heartbeat_runs_settled_process_identity_idx", + "columns": [ + { + "expression": "process_host_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' and \"heartbeat_runs\".\"status\" in ('failed', 'cancelled', 'timed_out', 'interrupted')\n and \"heartbeat_runs\".\"finished_at\" is not null\n and (\"heartbeat_runs\".\"process_pid\" is not null or \"heartbeat_runs\".\"process_group_id\" is not null or \"heartbeat_runs\".\"process_started_at\" is not null)", + "concurrently": true, + "method": "btree", + "with": {} + }, + "heartbeat_runs_pending_api_stop_idx": { + "name": "heartbeat_runs_pending_api_stop_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"status\" = 'cancelled' and \"heartbeat_runs\".\"driver_kind\" = 'cursor_cloud_api'\n and (\"heartbeat_runs\".\"result_json\"->'executionCancellation'->>'state') = 'requested'", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_company_created_at_desc_idx": { "name": "heartbeat_runs_company_created_at_desc_idx", "columns": [ @@ -59607,6 +59784,12 @@ "primaryKey": false, "notNull": false }, + "progress": { + "name": "progress", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -60113,6 +60296,277 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.tool_mcp_elicitations": { + "name": "tool_mcp_elicitations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "agent_id": { + "name": "agent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "invocation_id": { + "name": "invocation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "grant_id": { + "name": "grant_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "catalog_tool_name": { + "name": "catalog_tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "call_identity": { + "name": "call_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "era": { + "name": "era", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sealed_request_state": { + "name": "sealed_request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "questions": { + "name": "questions", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "arguments_identity": { + "name": "arguments_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "claimed_until": { + "name": "claimed_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sent_at": { + "name": "sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tool_mcp_elicitations_call_idx": { + "name": "tool_mcp_elicitations_call_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "call_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_invocation_idx": { + "name": "tool_mcp_elicitations_invocation_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "invocation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tool_mcp_elicitations_waiting_uq": { + "name": "tool_mcp_elicitations_waiting_uq", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"tool_mcp_elicitations\".\"status\" = 'waiting'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tool_mcp_elicitations_company_id_companies_id_fk": { + "name": "tool_mcp_elicitations_company_id_companies_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk": { + "name": "tool_mcp_elicitations_invocation_id_tool_invocations_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_invocations", + "columnsFrom": [ + "invocation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tool_mcp_elicitations_connection_id_tool_connections_id_fk": { + "name": "tool_mcp_elicitations_connection_id_tool_connections_id_fk", + "tableFrom": "tool_mcp_elicitations", + "tableTo": "tool_connections", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.tool_mcp_gateway_tokens": { "name": "tool_mcp_gateway_tokens", "schema": "", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index f3774187bbb..9790c0d8fc3 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3396,6 +3396,13 @@ "when": 1794623177597, "tag": "0486_heartbeat_runs_company_finished_index", "breakpoints": true + }, + { + "idx": 487, + "version": "7", + "when": 1794639469687, + "tag": "0487_heartbeat_runs_settled_process_indexes", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/heartbeat_runs.ts b/packages/db/src/schema/heartbeat_runs.ts index 0b424df776d..72ce4e56c22 100644 --- a/packages/db/src/schema/heartbeat_runs.ts +++ b/packages/db/src/schema/heartbeat_runs.ts @@ -250,6 +250,16 @@ export const heartbeatRuns = pgTable( ), companyFinishedIdx: index("heartbeat_runs_company_finished_idx").on(table.companyId, table.finishedAt.desc()) .where(sql`${table.finishedAt} is not null`).concurrently(), + settledProcessIdentityIdx: index("heartbeat_runs_settled_process_identity_idx") + .on(table.processHostHash, table.companyId, table.id) + .where(sql`${table.runtimeMode} = 'native' and ${table.status} in ('failed', 'cancelled', 'timed_out', 'interrupted') + and ${table.finishedAt} is not null + and (${table.processPid} is not null or ${table.processGroupId} is not null or ${table.processStartedAt} is not null)`) + .concurrently(), + pendingApiStopIdx: index("heartbeat_runs_pending_api_stop_idx").on(table.companyId, table.id) + .where(sql`${table.status} = 'cancelled' and ${table.driverKind} = 'cursor_cloud_api' + and (${table.resultJson}->'executionCancellation'->>'state') = 'requested'`) + .concurrently(), companyCreatedAtDescIdx: index("heartbeat_runs_company_created_at_desc_idx").on( table.companyId, table.createdAt.desc(), diff --git a/server/src/__tests__/settled-native-process-rechecks.test.ts b/server/src/__tests__/settled-native-process-rechecks.test.ts new file mode 100644 index 00000000000..b4616bcfe9d --- /dev/null +++ b/server/src/__tests__/settled-native-process-rechecks.test.ts @@ -0,0 +1,296 @@ +import { randomUUID } from "node:crypto"; +import os from "node:os"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { setImmediate as immediate } from "node:timers/promises"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, applyPendingMigrations, authUsers, closeCheckpointDb, closeLivenessDb, companies, companyMemberships, + controllerInstances, createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { meterDatabaseStatements } from "./helpers/database-statement-meter.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { notifyLiveRunRecovery } from "../services/live-run-recovery-notifications.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-lifecycle.js"; +import { discoverSettledNativeRunProcesses } from "../services/native-local-process-stop.js"; +import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; + +vi.setConfig({ testTimeout: 300_000, hookTimeout: 180_000 }); + +describe("settled native process re-checks", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let meter: ReturnType; + let heartbeat: ReturnType; + let coordinator: ReturnType; + const errors: unknown[] = []; + const passes = new Set>(); + const children: number[] = []; + const checks = new Map(); + const stops: Array<() => Promise> = []; + + beforeAll(async () => { + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_SETTLED_RECHECK_TEST_DATABASE_URL"); + if (!external) temporary = await startEmbeddedPostgresTestDatabase("settled-recheck-"); + else await applyPendingMigrations(external); + const url = external ?? temporary!.connectionString; + db = createDb(url, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + meter = meterDatabaseStatements(db, url); + await Promise.all(Array.from({ length: 4 }, () => db.execute(sql`select pg_sleep(0.01)`))); + db.$client.options.debug = (_connection, query, parameters) => { + if (!/for update/i.test(query) || !query.includes("sessionCheckpoint")) return; + const runId = (parameters as unknown[]).find(value => typeof value === "string" && checks.has(value)) as string | undefined; + if (runId) checks.set(runId, checks.get(runId)! + 1); + }; + }); + beforeEach(async () => { + await db.execute(sql`truncate companies, controller_instances cascade`); + errors.length = 0; checks.clear(); + heartbeat = heartbeatService(db); + coordinator = newCoordinator(); + }); + afterEach(async () => { + for (const stop of stops.splice(0)) await stop(); + await coordinator.stop(); await heartbeat.drainActiveRunExecutions(); + for (const pid of children.splice(0)) { try { process.kill(-pid, "SIGKILL"); } catch { } } + vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); + }); + afterAll(async () => { + meter?.uninstall(); + if (db) { await closeLivenessDb(db, 1); await closeCheckpointDb(db, 1); } + await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); + }); + + function newCoordinator() { + const created = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, + onError: error => errors.push(error) }); + const register = created.register.bind(created); + created.register = (queue, task) => register(queue, { ...task, run: signal => { + const pass = Promise.resolve(task.run(signal)); passes.add(pass); + void pass.finally(() => passes.delete(pass)); return pass; + } }); + return created; + } + + async function fixture() { + const companyId = randomUUID(), actorId = "settled-recheck-user"; + await db.insert(authUsers).values({ id: actorId, name: "Owner", email: `${actorId}@example.test`, + createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companies).values({ id: companyId, name: "Settled", issuePrefix: `SR${companyId.slice(0, 6)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: actorId }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId, + status: "active", membershipRole: "owner" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Worker", status: "idle", adapterType: "process", + adapterConfig: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Work", status: "done", assigneeAgentId: agent.id }).returning(); + return { companyId, agentId: agent.id, issueId: issue.id }; + } + type Fixture = Awaited>; + + async function exitedPid() { + const child = spawn(process.execPath, ["-e", "0"], { detached: true, stdio: "ignore" }); + const pid = child.pid!; await once(child, "exit"); return pid; + } + function livePid() { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + child.unref(); children.push(child.pid!); return child.pid!; + } + async function kill(pid: number) { + try { process.kill(-pid, "SIGKILL"); } catch { } + for (let attempt = 0; attempt < 100; attempt++) { + try { process.kill(pid, 0); } catch { return; } + await new Promise(resolve => setImmediate(resolve)); + } + } + + type Seed = { host?: { hostname: string; hash: string | null }; phase?: string | null; checkpoint?: boolean; live?: boolean; + birthMismatch?: boolean; remote?: "stopped" | "unconfirmed" }; + async function seed(f: Fixture, options: Seed = {}) { + const pid = options.live ? livePid() : await exitedPid(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, runtimeMode: "native", + nativeIssueId: f.issueId, status: "running", invocationSource: "assignment", triggerDetail: "system", + runnerProfileJson: options.checkpoint === false ? {} : { sessionCheckpoint: { providerSessionId: "thread" } } }).returning(); + if (options.host || options.birthMismatch) { + const startedAt = new Date(); + const host = options.host ?? { hostname: os.hostname(), hash: null }; + await appendHeartbeatRunEvent(db, { companyId: f.companyId, runId: run.id, agentId: f.agentId, + eventType: "native.process_identity_recorded", stream: "system", level: "info", message: "identity", + payload: { targetKind: options.remote ? "remote" : "local", hostname: host.hostname, processPid: pid, processGroupId: pid, + processStartedAt: new Date(startedAt.getTime() - (options.birthMismatch ? 5_000 : 0)).toISOString() }, + runUpdate: { processPid: pid, processGroupId: pid, processStartedAt: startedAt, processHostHash: host.hash } }); + } else { + await persistHeartbeatRunProcessMetadata(db, run.id, { pid, processGroupId: pid, startedAt: new Date().toISOString(), targetKind: "local" }); + } + if (options.phase !== null) await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId, + runId: run.id, phase: options.phase ?? "committed", controllerBootId: randomUUID() }); + const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId, heartbeatRunId: run.id, + provider: options.remote ? "daytona" : "local", providerLeaseId: options.remote ? randomUUID() : null, leasePolicy: "ephemeral", + status: "released", releasedAt: new Date(), cleanupStatus: "success" }).returning(); + if (options.remote === "stopped") await db.update(environmentLeases).set({ metadata: { + remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" }), + } }).where(eq(environmentLeases.id, lease.id)); + await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + checks.set(run.id, 0); + return { runId: run.id, pid }; + } + + async function start(target = coordinator) { + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await heartbeat.registerResidualRecoveryWork(target, () => true).ready; + await advance(2_000); + } + async function advance(ms: number) { + const until = Date.now() + ms; + while (Date.now() < until) { + await vi.advanceTimersByTimeAsync(Math.min(1_000, until - Date.now())); + await Promise.allSettled([...passes]); await immediate(); + } + } + const companyHint = (companyId: string) => db.transaction(tx => notifyLiveRunRecovery(tx, { companyId })); + const runHint = (companyId: string, runId: string) => db.transaction(tx => notifyLiveRunRecovery(tx, { companyId, runId })); + async function hints(companyId: string, count: number) { + for (let index = 0; index < count; index++) { await companyHint(companyId); await advance(31_000); } + } + const processPid = async (id: string) => (await db.select({ pid: heartbeatRuns.processPid }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, id)))[0]!.pid; + const stopReceipts = async (id: string) => (await db.select().from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.runId, id), eq(heartbeatRunEvents.eventType, "native.local_process_stopped")))).length; + + it.each([ + ["recorded by a replaced container", { host: { hostname: "replaced-container", hash: "replaced-container-space" } }], + ["whose coordinator phase is unsettled", { phase: "workspace_finalizing" }], + ["without a checkpoint or provider evidence", { checkpoint: false }], + ] as const)("never re-reads a terminal run %s on company hints", async (_name, options) => { + const f = await fixture(); + const { runId, pid } = await seed(f, options as Seed); + await start(); + await hints(f.companyId, 4); + expect(checks.get(runId)).toBe(0); + expect(await processPid(runId)).toBe(pid); + }); + + it.each(["live", "stopped"] as const)("leaves a %s older server's runs to that server", async state => { + const f = await fixture(); + const bootId = randomUUID(); + await db.insert(controllerInstances).values({ bootId, processSpace: "older-server-space", pid: 4242, + leaseExpiresAt: new Date(Date.now() + (state === "live" ? 60_000 : -600_000)), + stoppedAt: state === "stopped" ? new Date() : null }); + const { runId, pid } = await seed(f, { host: { hostname: "older-server", hash: "older-server-space" } }); + await db.update(nativeRunFinalizations).set({ controllerBootId: bootId, controllerHostHash: "older-server-space" }) + .where(eq(nativeRunFinalizations.runId, runId)); + const own = await seed(f); + await start(); + expect(await processPid(own.runId)).toBeNull(); + await hints(f.companyId, 2); + expect(checks.get(runId)).toBe(0); + expect(await processPid(runId)).toBe(pid); + expect(await stopReceipts(runId)).toBe(0); + }); + + it("re-admits a run once its coordinator phase settles", async () => { + const f = await fixture(); + const { runId } = await seed(f, { phase: "workspace_finalizing" }); + await start(); + await hints(f.companyId, 1); + expect(await processPid(runId)).not.toBeNull(); + await db.update(nativeRunFinalizations).set({ phase: "committed" }).where(eq(nativeRunFinalizations.runId, runId)); + await hints(f.companyId, 1); + expect(await processPid(runId)).toBeNull(); + expect(checks.get(runId)).toBe(1); + expect(await stopReceipts(runId)).toBe(1); + }); + + it("re-admits a run once late provider evidence arrives", async () => { + const f = await fixture(); + const { runId } = await seed(f, { checkpoint: false }); + await start(); + await hints(f.companyId, 1); + expect(await processPid(runId)).not.toBeNull(); + await appendHeartbeatRunEvent(db, { companyId: f.companyId, runId, agentId: f.agentId, eventType: "harness.ready", + stream: "system", level: "info", message: "ready" }); + await hints(f.companyId, 1); + expect(await processPid(runId)).toBeNull(); + expect(checks.get(runId)).toBe(1); + }); + + it("releases legacy rows without a PID-space hash only when this host recorded them", async () => { + const f = await fixture(); + const own = await seed(f, { host: { hostname: os.hostname(), hash: null } }); + const foreign = await seed(f, { host: { hostname: "legacy-other-host", hash: null } }); + await start(); + expect(await processPid(own.runId)).toBeNull(); + await hints(f.companyId, 2); + expect(checks.get(foreign.runId)).toBe(0); + expect(await processPid(foreign.runId)).toBe(foreign.pid); + }); + + it("releases a sandbox run from any server once its termination receipt exists", async () => { + const f = await fixture(); + const pending = await seed(f, { host: { hostname: "replaced-container", hash: "remote" }, remote: "unconfirmed" }); + const stopped = await seed(f, { host: { hostname: "replaced-container", hash: "remote" }, remote: "stopped" }); + await start(); + expect(await processPid(stopped.runId)).toBeNull(); + expect(await processPid(pending.runId)).not.toBeNull(); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, pending.runId)); + await db.update(environmentLeases).set({ metadata: { + remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" }), + } }).where(eq(environmentLeases.id, lease.id)); + await runHint(f.companyId, pending.runId); + await advance(2_000); + expect(await processPid(pending.runId)).toBeNull(); + }); + + it("backs off a live local process and releases it on the next run-scoped hint after it exits", async () => { + const f = await fixture(); + const { runId, pid } = await seed(f, { live: true }); + await start(); + expect(checks.get(runId)).toBe(1); + await hints(f.companyId, 8); + expect(checks.get(runId)).toBeLessThanOrEqual(4); + expect(await processPid(runId)).toBe(pid); + await kill(pid); + await runHint(f.companyId, runId); + await advance(2_000); + expect(await processPid(runId)).toBeNull(); + expect(await stopReceipts(runId)).toBe(1); + }); + + it("checks a same-host skip once with one locking statement, then only on its own trigger", async () => { + const f = await fixture(); + const { runId } = await seed(f, { birthMismatch: true, host: { hostname: os.hostname(), hash: null } }); + meter.startLog(); + await start(); + const log = meter.stopLog(); + expect(checks.get(runId)).toBe(1); + const inCheck = log.filter(entry => entry.inTransaction || /^\s*begin/i.test(entry.sql)); + expect(inCheck.filter(entry => /for update/i.test(entry.sql))).toHaveLength(1); + expect(inCheck.length).toBeLessThanOrEqual(3); + await hints(f.companyId, 3); + expect(checks.get(runId)).toBe(1); + await runHint(f.companyId, runId); + await advance(2_000); + expect(checks.get(runId)).toBe(2); + }); + + it("checks a pending run at most once after a restart and discovers no excluded run at startup", async () => { + const f = await fixture(); + const { runId } = await seed(f, { live: true }); + for (let index = 0; index < 50; index++) await seed(f, { host: { hostname: "replaced-container", hash: "replaced-container-space" } }); + await start(); + expect(checks.get(runId)).toBe(1); + await coordinator.stop(); + const restarted = newCoordinator(); + stops.push(() => restarted.stop()); + await start(restarted); + expect(checks.get(runId)).toBe(2); + vi.useRealTimers(); + const before = meter.read(); + const counted = [...checks.values()].reduce((sum, value) => sum + value, 0); + await discoverSettledNativeRunProcesses(db, { skip: id => id === runId }); + expect(meter.read().statements - before.statements).toBe(1); + expect([...checks.values()].reduce((sum, value) => sum + value, 0)).toBe(counted); + }); +}); diff --git a/server/src/services/heartbeat/live-recovery-work.ts b/server/src/services/heartbeat/live-recovery-work.ts index 4c2ec0d6376..07bc8d14950 100644 --- a/server/src/services/heartbeat/live-recovery-work.ts +++ b/server/src/services/heartbeat/live-recovery-work.ts @@ -1,6 +1,6 @@ import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js"; import { legacyWorkspaceRecoveryNotMigrated } from "../legacy-workspace-recovery-migration.js"; -import { settledNativeProcessIdentityCondition } from "../native-local-process-stop.js"; +import { settledNativeProcessCandidates } from "../native-local-process-stop.js"; import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; import { agents, environmentLeases, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; @@ -9,6 +9,8 @@ import type { LiveResourceRecoveryScope, LiveRunRecoveryScope } from "../live-ru import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; const TERMINAL = ["succeeded", "failed", "cancelled", "timed_out", "interrupted"]; +const SETTLED_PENDING_BASE_DELAY_MS = 30_000; +const SETTLED_PENDING_MAX_DELAY_MS = 15 * 60_000; type Scope = LiveRunRecoveryScope & LiveResourceRecoveryScope; type Entry = { id: string; companyId: string; runId?: string | null; dueAt: number }; type CheckScope = { companyId: string; runId?: string; leaseId?: string; agentId?: string; @@ -27,6 +29,8 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType boolean) { function register(queue: DeliveryQueue, kind: RunKind | "orphanLease" | "shutdown" | "agent") { const cursors = new Map(); + const settledOutcomes = new Map(); + const settledScans = new Map>(); let enabled = false; async function load(scope?: Scope) { const key = scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId, @@ -63,12 +67,11 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType)?.deadlineAt)) || Date.now(), })); + } else if (kind === "settled") { + const page = await loadSettled(scope, cursor); + if (page.complete) cursors.delete(key); else cursors.set(key, page.rows.at(-1)!.id); + return { complete: page.complete, rows: applySettledOutcomes(key, scope, page.rows, page.complete) }; } else { - // Retirement may commit before Stop acknowledgement. This is only a - // work hint; the owner rechecks the complete provider/process proof. - const pendingApiStop = and(eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.driverKind, "cursor_cloud_api"), - sql`${heartbeatRuns.runnerProfileJson}->>'backend' = 'cursor_cloud_api'`, - sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`); const candidates = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, nextAttemptAt: nativeRunFinalizations.nextAttemptAt, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }) .from(heartbeatRuns).leftJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id), @@ -83,12 +86,8 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType'nativeQuestionCancellation' is not null`) - : kind === "active" ? and(eq(heartbeatRuns.status, "running"), - sql`${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is null`) - : and(eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["failed", "cancelled", "timed_out", "interrupted"]), - isNotNull(heartbeatRuns.finishedAt), or(pendingApiStop, - and(or(isNotNull(heartbeatRuns.processPid), isNotNull(heartbeatRuns.processGroupId), - isNotNull(heartbeatRuns.processStartedAt)), settledNativeProcessIdentityCondition(db)))), + : and(eq(heartbeatRuns.status, "running"), + sql`${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is null`), )).orderBy(asc(heartbeatRuns.id)).limit(100); rows = candidates.map(row => ({ id: row.id, companyId: row.companyId, dueAt: kind === "native" ? Math.max(Date.now(), row.nextAttemptAt?.getTime() ?? 0, row.leaseExpiresAt?.getTime() ?? 0) : Date.now() })); @@ -97,6 +96,50 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType) => db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId }) + .from(heartbeatRuns).where(and(bounds, condition)).orderBy(asc(heartbeatRuns.id)).limit(100); + const processes = await settledNativeProcessCandidates(db, { companyId: scope?.companyId, runId: scope?.runId, + after: cursor, limit: 100 }); + const stops = await select(and(eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.driverKind, "cursor_cloud_api"), + sql`(${heartbeatRuns.resultJson}->'executionCancellation'->>'state') = 'requested'`, + eq(heartbeatRuns.runtimeMode, "native"), isNotNull(heartbeatRuns.finishedAt), + sql`${heartbeatRuns.runnerProfileJson}->>'backend' = 'cursor_cloud_api'`)); + const merged = [...new Map([...processes, ...stops].map(row => [row.id, row])).values()] + .sort((left, right) => left.id < right.id ? -1 : left.id > right.id ? 1 : 0); + const complete = processes.length < 100 && stops.length < 100 && merged.length <= 100; + return { complete, rows: merged.slice(0, 100).map(row => ({ ...row, dueAt: Date.now() })) }; + } + function applySettledOutcomes(key: string, scope: Scope | undefined, rows: Entry[], complete: boolean) { + if (scope?.runId) settledOutcomes.delete(scope.runId); + const seen = settledScans.get(key) ?? new Set(); + for (const row of rows) seen.add(row.id); + if (!complete) settledScans.set(key, seen); + else { + settledScans.delete(key); + if (!scope?.runId) for (const [id, outcome] of settledOutcomes) + if ((!scope || outcome.companyId === scope.companyId) && !seen.has(id)) settledOutcomes.delete(id); + } + return rows.flatMap(row => { + const outcome = settledOutcomes.get(row.id); + if (outcome?.parked) return []; + return [{ ...row, dueAt: Math.max(row.dueAt, outcome?.notBefore ?? 0) }]; + }); + } + function recordSettledOutcome(entry: Entry, outcome: unknown): Entry | null | undefined { + if (outcome === "released") { settledOutcomes.delete(entry.id); return null; } + if (outcome === "skipped") { + settledOutcomes.set(entry.id, { companyId: entry.companyId, parked: true, attempts: 0, notBefore: 0 }); + return null; + } + if (outcome !== "pending") return undefined; + const attempts = (settledOutcomes.get(entry.id)?.attempts ?? 0) + 1; + const notBefore = Date.now() + Math.min(SETTLED_PENDING_MAX_DELAY_MS, SETTLED_PENDING_BASE_DELAY_MS * 2 ** (attempts - 1)); + settledOutcomes.set(entry.id, { companyId: entry.companyId, parked: false, attempts, notBefore }); + return { ...entry, dueAt: notBefore }; + } return registerScopedDeadlineWork(coordinator, { queue, reconstructionIntervalMs: 30_000, @@ -114,10 +157,12 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType row.id === entry.id); - if (!current) return null; - if (current.dueAt > Date.now()) return current; + if (kind !== "settled") { + const loaded = await load(currentScope); + const current = (Array.isArray(loaded) ? loaded : loaded.rows).find(row => row.id === entry.id); + if (!current) return null; + if (current.dueAt > Date.now()) return current; + } let next: number | null = null; const errors: unknown[] = []; const scope: CheckScope = { companyId: entry.companyId, @@ -126,8 +171,12 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType { next = next === null ? at : Math.min(next, at); }, onError: error => { errors.push(error); }, }; - await owner[kind](scope); + const outcome = await owner[kind](scope); if (errors.length) throw new AggregateError(errors, `Recovery obligation ${entry.id} failed`); + if (kind === "settled") { + const settled = recordSettledOutcome(entry, outcome); + if (settled !== undefined) return settled; + } if (kind === "native") { const refreshed = await load(currentScope); const remaining = (Array.isArray(refreshed) ? refreshed : refreshed.rows).find(row => row.id === entry.id); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 191f1ef963b..b2a866808bd 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -39,7 +39,7 @@ import { hasStopOnlyCleanup, settleStopOnlyCleanup, } from "../sandbox-stop-and-retain.js"; -import { discoverSettledNativeRunProcesses, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, recordNativeLocalProcessStopInTransaction, releaseSettledNativeRunProcess, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js"; +import { discoverSettledNativeRunProcesses, inspectSettledNativeRunProcess, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, recordNativeLocalProcessStopInTransaction, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js"; import { pendingCaseHandoffForRun } from "./case-handoff-lifecycle.js"; import { portableTaskPreparationLeaseId } from "../plugin-task-ownership.js"; import { retireInheritedRemoteRunnerIdentity } from "../native-runtime/native-inherited-runner-retry.js"; @@ -3109,10 +3109,11 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD if (!scope.runId || activeRunExecutions.has(scope.runId) || runningProcesses.has(scope.runId) || adapterExecutionControls.has(scope.runId)) { scope.onDeferredDeadline(Date.now() + 30_000); - return; + return "deferred"; } - const outcome = await releaseSettledNativeRunProcess(db, scope.runId, scope); - if (outcome !== "pending") await acknowledgeRemoteStop(scope.runId, scope.companyId); + const { outcome, remoteStop } = await inspectSettledNativeRunProcess(db, scope.runId, scope); + if (outcome !== "pending" && remoteStop) await acknowledgeRemoteStop(scope.runId, scope.companyId); + return outcome; }, orphanLease: scope => sweepOrphanedActiveLeases({ ...scope, backoffMs: 5 * 60_000 }), shutdown: scope => sweepLegacyShutdownWorkspaceSettlements(db, new Date(), scope), diff --git a/server/src/services/native-local-process-stop.ts b/server/src/services/native-local-process-stop.ts index 479d524c611..7c7a5e39546 100644 --- a/server/src/services/native-local-process-stop.ts +++ b/server/src/services/native-local-process-stop.ts @@ -1,13 +1,13 @@ import { notifyLiveRunRecovery } from "./live-run-recovery-notifications.js"; -import { and, asc, desc, eq, exists, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; -import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { and, desc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; import { notifyRetainedNativeSessionCleanup } from "./retained-native-session-cleanup-notifications.js"; import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -import { nativeSessionProviderEvidence } from "./native-runtime/native-session-resume.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { NATIVE_SESSION_PROVIDER_EVENT_TYPES } from "./native-runtime/native-session-resume.js"; import os from "node:os"; -import { currentProcessSpaceHash } from "./controller-process-identity.js"; +import { currentProcessSpaceHash, UNKNOWN_PROCESS_SPACE } from "./controller-process-identity.js"; import { proveOwnerGone } from "./owner-proof.js"; export const PROCESS_START_REQUESTED = "native.process_start_requested"; @@ -69,6 +69,12 @@ export async function recordNativeLocalProcessStop(db: Db, run: Pick lease.provider !== "local")) return false; if ((run.processPid && !absent(run.processPid)) || (run.processGroupId && !absent(-run.processGroupId))) return false; + await appendLocalProcessStop(db, { ...run, processStartedAt: run.processStartedAt?.toISOString() ?? null }); + return true; +} + +async function appendLocalProcessStop(db: Db, run: { id: string; companyId: string; agentId: string; + processPid: number | null; processGroupId: number | null; processStartedAt: string | null }) { await notifyDeliveryWork(db, DELIVERY_QUEUES.executionReplacement, JSON.stringify({ companyId: run.companyId, runId: run.id })); await appendHeartbeatRunEvent(db, { @@ -79,10 +85,8 @@ export async function recordNativeLocalProcessStop(db: Db, run: Pick 'local'`))), - and(gt(heartbeatRuns.processPid, 1), eq(heartbeatRuns.processPid, heartbeatRuns.processGroupId), - exists(db.select({ payload: identity.payload }).from(identity).where(and( - sql`${identity.payload}->>'targetKind' = 'local'`, sql`${identity.payload}->>'hostname' = ${os.hostname()}`, - sql`${identity.payload}->>'processPid' = ${heartbeatRuns.processPid}::text`, - sql`${identity.payload}->>'processGroupId' = ${heartbeatRuns.processGroupId}::text`, - ))))); +const providerEventTypes = () => sql.join(NATIVE_SESSION_PROVIDER_EVENT_TYPES.map(type => sql`${type}`), sql`, `); + +function settledNativeProcessReleaseCondition() { + const processSpace = currentProcessSpaceHash(); + const ownSpace = processSpace === UNKNOWN_PROCESS_SPACE ? undefined : eq(heartbeatRuns.processHostHash, processSpace); + const remoteLease = sql`exists (select 1 from environment_leases settled_lease where settled_lease.company_id = ${heartbeatRuns.companyId} + and settled_lease.heartbeat_run_id = ${heartbeatRuns.id} and settled_lease.provider is distinct from 'local')`; + const unrecordedSpace = or(isNull(heartbeatRuns.processHostHash), eq(heartbeatRuns.processHostHash, UNKNOWN_PROCESS_SPACE)); + return and(eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, SETTLED_RUN_STATUSES), + isNotNull(heartbeatRuns.finishedAt), + or(isNotNull(heartbeatRuns.processPid), isNotNull(heartbeatRuns.processGroupId), isNotNull(heartbeatRuns.processStartedAt)), + or(ownSpace, eq(heartbeatRuns.processHostHash, "remote"), isNull(heartbeatRuns.processHostHash), + eq(heartbeatRuns.processHostHash, UNKNOWN_PROCESS_SPACE)), + or(ownSpace, remoteLease, and(unrecordedSpace, sql`(select settled_identity.payload->>'hostname' from heartbeat_run_events settled_identity + where settled_identity.company_id = ${heartbeatRuns.companyId} and settled_identity.run_id = ${heartbeatRuns.id} + and settled_identity.event_type = ${PROCESS_IDENTITY_RECORDED} and settled_identity.source_event_id is null + order by settled_identity.seq desc limit 1) = ${os.hostname()}::text`)), + sql`not exists (select 1 from native_run_finalizations settled_coordinator + where settled_coordinator.run_id = ${heartbeatRuns.id} and settled_coordinator.phase not in ('committed', 'terminal_failure'))`, + sql`(coalesce(${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint', 'null'::jsonb) <> 'null'::jsonb + or exists (select 1 from heartbeat_run_events settled_evidence where settled_evidence.company_id = ${heartbeatRuns.companyId} + and settled_evidence.run_id = ${heartbeatRuns.id} and settled_evidence.event_type in (${providerEventTypes()})))`); +} + +export async function settledNativeProcessCandidates(db: Pick, input: { + companyId?: string; runId?: string; after?: string | null; limit: number; +}): Promise> { + const processSpace = currentProcessSpaceHash(); + const condition = and(input.companyId ? eq(heartbeatRuns.companyId, input.companyId) : undefined, + input.runId ? eq(heartbeatRuns.id, input.runId) : undefined, input.after ? gt(heartbeatRuns.id, input.after) : undefined, + settledNativeProcessReleaseCondition()); + const spaces = [processSpace === UNKNOWN_PROCESS_SPACE ? null : eq(heartbeatRuns.processHostHash, processSpace), + eq(heartbeatRuns.processHostHash, "remote"), eq(heartbeatRuns.processHostHash, UNKNOWN_PROCESS_SPACE), + isNull(heartbeatRuns.processHostHash)].filter(space => space !== null); + const branches = spaces.map(space => sql`(select ${heartbeatRuns.id} as id, ${heartbeatRuns.companyId} as company_id + from ${heartbeatRuns} where ${and(space, condition)} offset 0)`); + return await db.execute(sql`select id, company_id as "companyId" from (${sql.join(branches, sql` union all `)}) candidates + order by id limit ${input.limit}`) as unknown as Array<{ id: string; companyId: string }>; +} + +type SettledProcessFacts = { + companyId: string; agentId: string; runtimeMode: string; status: string; finished: boolean; + processPid: number | null; processGroupId: number | null; processStartedAt: string | null; apiDriver: boolean | null; + hasSessionCheckpoint: boolean; phase: string | null; providerEvidence: boolean; remoteLeases: boolean; + localLeasePending: boolean; identityMatches: boolean | null; +}; + +function readSettledProcessFacts(tx: Db, runId: string) { + return tx.execute(sql`select r.company_id as "companyId", r.agent_id as "agentId", r.runtime_mode as "runtimeMode", + r.status, r.finished_at is not null as "finished", r.process_pid as "processPid", r.process_group_id as "processGroupId", + to_char(r.process_started_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') as "processStartedAt", + r.driver_kind = 'cursor_cloud_api' as "apiDriver", + coalesce(r.runner_profile_json->'sessionCheckpoint', 'null'::jsonb) <> 'null'::jsonb as "hasSessionCheckpoint", + (select f.phase from native_run_finalizations f where f.run_id = r.id limit 1) as "phase", + exists (select 1 from heartbeat_run_events e where e.company_id = r.company_id and e.run_id = r.id + and e.event_type in (${providerEventTypes()})) as "providerEvidence", + exists (select 1 from environment_leases l where l.company_id = r.company_id and l.heartbeat_run_id = r.id + and l.provider is distinct from 'local') as "remoteLeases", + exists (select 1 from environment_leases l where l.company_id = r.company_id and l.heartbeat_run_id = r.id + and (l.released_at is null or l.status = 'pending_cleanup' or l.cleanup_status = 'failed')) as "localLeasePending", + (select identity.payload->>'targetKind' = 'local' and identity.payload->>'hostname' = ${os.hostname()}::text + and identity.payload ? 'processPid' and identity.payload->'processPid' = coalesce(to_jsonb(r.process_pid), 'null'::jsonb) + and identity.payload ? 'processGroupId' and identity.payload->'processGroupId' = coalesce(to_jsonb(r.process_group_id), 'null'::jsonb) + and case when r.process_started_at is not null + then identity.payload->>'processStartedAt' = to_char(r.process_started_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') + else coalesce(r.process_pid > 1 and r.process_group_id = r.process_pid, false) end + from (select e.payload from heartbeat_run_events e where e.company_id = r.company_id and e.run_id = r.id + and e.event_type = ${PROCESS_IDENTITY_RECORDED}::text and e.source_event_id is null order by e.seq desc limit 1) identity + ) as "identityMatches" + from heartbeat_runs r where r.id = ${runId}::uuid for update of r`) as unknown as Promise; } export async function releaseSettledNativeRunProcess(db: Db, runId: string, options: { onDeferredDeadline?: (at: number) => void } = {}): Promise { + return (await inspectSettledNativeRunProcess(db, runId, options)).outcome; +} + +export async function inspectSettledNativeRunProcess(db: Db, runId: string, options: { onDeferredDeadline?: (at: number) => void } = {}): Promise { + let remoteStop = false; const outcome = await db.transaction(async transaction => { const tx = transaction as unknown as Db; - const [run] = await tx.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, - runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, finishedAt: heartbeatRuns.finishedAt, - processPid: heartbeatRuns.processPid, processGroupId: heartbeatRuns.processGroupId, processStartedAt: heartbeatRuns.processStartedAt, - hasSessionCheckpoint: sql`coalesce(${heartbeatRuns.runnerProfileJson}->'sessionCheckpoint', 'null'::jsonb) <> 'null'::jsonb`, - }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update").limit(1); - if (!run || run.runtimeMode !== "native" || !SETTLED_RUN_STATUSES.includes(run.status) || !run.finishedAt || + const [run] = await readSettledProcessFacts(tx, runId); + if (!run) return "skipped"; + remoteStop = run.remoteLeases || (run.status === "cancelled" && run.apiDriver === true); + if (run.runtimeMode !== "native" || !SETTLED_RUN_STATUSES.includes(run.status) || !run.finished || (run.processPid === null && run.processGroupId === null && run.processStartedAt === null)) return "skipped"; - const [coordinator] = await tx.select({ phase: nativeRunFinalizations.phase }).from(nativeRunFinalizations) - .where(eq(nativeRunFinalizations.runId, run.id)).limit(1); - if (coordinator && !SETTLED_COORDINATOR_PHASES.includes(coordinator.phase)) return "skipped"; - if (!run.hasSessionCheckpoint && - !(await nativeSessionProviderEvidence(tx, [run.id])).has(run.id)) return "skipped"; - const leases = await tx.select().from(environmentLeases).where(and( - eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), - )); - if (leases.some(lease => lease.provider !== "local")) { + if (run.phase !== null && !SETTLED_COORDINATOR_PHASES.includes(run.phase)) return "skipped"; + if (!run.hasSessionCheckpoint && !run.providerEvidence) return "skipped"; + if (run.remoteLeases) { + const leases = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, runId), + )); if (!leases.every(hasRemoteTerminationReceipt)) return "pending"; } else { - if (leases.some(lease => !lease.releasedAt || lease.status === "pending_cleanup" || lease.cleanupStatus === "failed")) return "pending"; - const [identity] = await tx.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents).where(and( - eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), - eq(heartbeatRunEvents.eventType, PROCESS_IDENTITY_RECORDED), isNull(heartbeatRunEvents.sourceEventId), - )).orderBy(desc(heartbeatRunEvents.seq)).limit(1); - if (identity?.payload?.targetKind !== "local" || identity.payload.hostname !== os.hostname() || - identity.payload.processPid !== run.processPid || identity.payload.processGroupId !== run.processGroupId || - (run.processStartedAt ? identity.payload.processStartedAt !== run.processStartedAt.toISOString() - : !Number.isSafeInteger(run.processPid) || run.processPid! <= 1 || run.processGroupId !== run.processPid)) return "skipped"; - if (!(await recordNativeLocalProcessStop(tx, run))) { + if (run.localLeasePending) return "pending"; + if (run.identityMatches !== true) return "skipped"; + if ((!run.processPid && !run.processGroupId) || (run.processPid && !absent(run.processPid)) + || (run.processGroupId && !absent(-run.processGroupId))) { options.onDeferredDeadline?.(Date.now() + 30_000); return "pending"; } + await appendLocalProcessStop(tx, { id: runId, companyId: run.companyId, agentId: run.agentId, + processPid: run.processPid, processGroupId: run.processGroupId, processStartedAt: run.processStartedAt }); } - await notifyLiveRunRecovery(tx, { companyId: run.companyId, runId: run.id, agentId: run.agentId }); - await notifyRetainedNativeSessionCleanup(tx, { companyId: run.companyId, runId: run.id }); + await notifyLiveRunRecovery(tx, { companyId: run.companyId, runId, agentId: run.agentId }); + await notifyRetainedNativeSessionCleanup(tx, { companyId: run.companyId, runId }); // Local and remote stops both close the run's usage source. await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); await tx.update(heartbeatRuns).set({ processPid: null, processGroupId: null, processStartedAt: null, updatedAt: new Date(), - }).where(eq(heartbeatRuns.id, run.id)); + }).where(eq(heartbeatRuns.id, runId)); return "released" as const; }); if (outcome !== "pending") pendingSettledProcessReleases.delete(runId); - return outcome; + return { outcome, remoteStop }; } const MAX_SETTLED_PROCESS_RELEASE_ATTEMPTS = 20; @@ -473,13 +530,7 @@ export async function discoverSettledNativeRunProcesses(db: Db, input: { let after: string | null = null; let released = 0; do { - const candidates: Array<{ id: string }> = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.runtimeMode, "native"), - inArray(heartbeatRuns.status, SETTLED_RUN_STATUSES), - isNotNull(heartbeatRuns.finishedAt), - or(isNotNull(heartbeatRuns.processPid), isNotNull(heartbeatRuns.processGroupId), isNotNull(heartbeatRuns.processStartedAt)), - ...(after ? [gt(heartbeatRuns.id, after)] : []), - )).orderBy(asc(heartbeatRuns.id)).limit(limit); + const candidates: Array<{ id: string }> = await settledNativeProcessCandidates(db, { after, limit }); for (const candidate of candidates) { if (input.skip?.(candidate.id)) continue; if (await settleNativeRunProcess(db, candidate.id) === "released") released += 1; diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 5b10620f53c..61c5ef5bc31 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -210,7 +210,7 @@ export function selectNativeSessionResumeRun< return null; } -const NATIVE_SESSION_PROVIDER_EVENT_TYPES = [ +export const NATIVE_SESSION_PROVIDER_EVENT_TYPES = [ "harness.ready", "session.started", "session.resumed",