diff --git a/packages/db/src/migrations/0442_low_salo.sql b/packages/db/src/migrations/0442_low_salo.sql new file mode 100644 index 00000000000..63037106b3c --- /dev/null +++ b/packages/db/src/migrations/0442_low_salo.sql @@ -0,0 +1 @@ +ALTER TABLE "heartbeat_runs" ADD COLUMN "process_host_hash" text; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0437_snapshot.json b/packages/db/src/migrations/meta/0442_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0437_snapshot.json rename to packages/db/src/migrations/meta/0442_snapshot.json index 96b47269974..2944c74ed51 100644 --- a/packages/db/src/migrations/meta/0437_snapshot.json +++ b/packages/db/src/migrations/meta/0442_snapshot.json @@ -1,6 +1,6 @@ { - "id": "2c0cb580-67fc-45d3-9ca5-cce4148dd1a4", - "prevId": "a80d63fe-5111-49e8-a39a-7a5589b3b31a", + "id": "b3f3d712-65da-4b5a-bbf5-3c22fe8aae40", + "prevId": "a2ec96c6-3cf1-4c51-8ee8-b35f090bfd42", "version": "7", "dialect": "postgresql", "tables": { @@ -34880,6 +34880,18 @@ "primaryKey": false, "notNull": false }, + "process_host_hash": { + "name": "process_host_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "legacy_process_settlement": { + "name": "legacy_process_settlement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "last_output_at": { "name": "last_output_at", "type": "timestamp with time zone", @@ -44457,6 +44469,256 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.native_provider_artifact_references": { + "name": "native_provider_artifact_references", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifact_references_artifact_idx": { + "name": "native_provider_artifact_references_artifact_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_run_idx": { + "name": "native_provider_artifact_references_run_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_backup_idx": { + "name": "native_provider_artifact_references_backup_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk": { + "name": "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_provider_artifacts", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "native_provider_artifact_references_backup_id_native_harness_backups_id_fk": { + "name": "native_provider_artifact_references_backup_id_native_harness_backups_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_harness_backups", + "columnsFrom": [ + "backup_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.native_provider_artifacts": { + "name": "native_provider_artifacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "closure_digest": { + "name": "closure_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "upload_token": { + "name": "upload_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "descriptor": { + "name": "descriptor", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reference": { + "name": "reference", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifacts_company_digest_idx": { + "name": "native_provider_artifacts_company_digest_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closure_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifacts\".\"status\" = 'ready'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.native_run_finalizations": { "name": "native_run_finalizations", "schema": "", @@ -44516,6 +44778,12 @@ "primaryKey": false, "notNull": false }, + "controller_host_hash": { + "name": "controller_host_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, "controller_process_started_at": { "name": "controller_process_started_at", "type": "timestamp with time zone", @@ -64075,6 +64343,139 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.mcp_oauth_client_changes": { + "name": "mcp_oauth_client_changes", + "schema": "", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_client_changes_revision_idx": { + "name": "mcp_client_changes_revision_idx", + "columns": [ + { + "expression": "revision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_client_change_revision": { + "name": "mcp_client_change_revision", + "value": "\"mcp_oauth_client_changes\".\"revision\" >= 1" + }, + "mcp_client_change_operation": { + "name": "mcp_client_change_operation", + "value": "\"mcp_oauth_client_changes\".\"operation\" IN ('upsert','delete')" + } + }, + "isRLSEnabled": false + }, + "public.mcp_oauth_client_mirror_state": { + "name": "mcp_oauth_client_mirror_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "remote_responsibility": { + "name": "remote_responsibility", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "device_cutover_stack_id": { + "name": "device_cutover_stack_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_cutover_at": { + "name": "device_cutover_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "receipt": { + "name": "receipt", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_client_mirror_singleton": { + "name": "mcp_client_mirror_singleton", + "value": "\"mcp_oauth_client_mirror_state\".\"id\" = 'public'" + }, + "mcp_client_mirror_revision": { + "name": "mcp_client_mirror_revision", + "value": "\"mcp_oauth_client_mirror_state\".\"revision\" >= 0" + }, + "mcp_device_cutover_identity": { + "name": "mcp_device_cutover_identity", + "value": "(\"mcp_oauth_client_mirror_state\".\"device_cutover_at\" IS NULL AND \"mcp_oauth_client_mirror_state\".\"device_cutover_stack_id\" IS NULL) OR (\"mcp_oauth_client_mirror_state\".\"device_cutover_at\" IS NOT NULL AND \"mcp_oauth_client_mirror_state\".\"device_cutover_stack_id\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, "public.mcp_oauth_clients": { "name": "mcp_oauth_clients", "schema": "", @@ -64143,6 +64544,13 @@ "primaryKey": false, "notNull": true }, + "ingress_owner": { + "name": "ingress_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'tenant'" + }, "device_code_hash": { "name": "device_code_hash", "type": "text", @@ -64298,7 +64706,12 @@ "compositePrimaryKeys": {}, "uniqueConstraints": {}, "policies": {}, - "checkConstraints": {}, + "checkConstraints": { + "mcp_device_ingress_owner": { + "name": "mcp_device_ingress_owner", + "value": "\"mcp_oauth_device_requests\".\"ingress_owner\" IN ('tenant','cloud')" + } + }, "isRLSEnabled": false }, "public.mcp_oauth_grants": { @@ -64361,6 +64774,19 @@ "primaryKey": false, "notNull": false }, + "revocation_revision": { + "name": "revocation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "revocation_index_receipt": { + "name": "revocation_index_receipt", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -64691,6 +65117,13 @@ "primaryKey": false, "notNull": false }, + "revocation_revision": { + "name": "revocation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, "expires_at": { "name": "expires_at", "type": "timestamp with time zone", @@ -64736,6 +65169,33 @@ "method": "btree", "with": {} }, + "mcp_oauth_tokens_revocation_revision_idx": { + "name": "mcp_oauth_tokens_revocation_revision_idx", + "columns": [ + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "revocation_revision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "mcp_oauth_tokens_expiry_idx": { "name": "mcp_oauth_tokens_expiry_idx", "columns": [ diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 08d4023c1f1..f4827d2bc44 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3081,6 +3081,13 @@ "when": 1794069983192, "tag": "0441_lucky_chameleon", "breakpoints": true + }, + { + "idx": 442, + "version": "7", + "when": 1794071961340, + "tag": "0442_low_salo", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/heartbeat_runs.ts b/packages/db/src/schema/heartbeat_runs.ts index 247624e48c0..239d0cad70e 100644 --- a/packages/db/src/schema/heartbeat_runs.ts +++ b/packages/db/src/schema/heartbeat_runs.ts @@ -96,6 +96,9 @@ export const heartbeatRuns = pgTable( processPid: integer("process_pid"), processGroupId: integer("process_group_id"), processStartedAt: timestamp("process_started_at", { withTimezone: true }), + // PID space of processPid (controller-process-identity hostHash), or + // "remote" for a sandbox PID. Null on rows saved before it existed. + processHostHash: text("process_host_hash"), legacyProcessSettlement: jsonb("legacy_process_settlement").$type>(), lastOutputAt: timestamp("last_output_at", { withTimezone: true }), lastOutputSeq: integer("last_output_seq").notNull().default(0), diff --git a/server/src/__tests__/live-recovery-work.test.ts b/server/src/__tests__/live-recovery-work.test.ts index 77c5714f882..eaade7ebc9c 100644 --- a/server/src/__tests__/live-recovery-work.test.ts +++ b/server/src/__tests__/live-recovery-work.test.ts @@ -512,7 +512,7 @@ describe("residual recovery production matrix", () => { } finally { db.$client.options.debug = prior; } expect(await getRun(run.id)).toMatchObject({ processPid: pid, processStartedAt: null }); await quiet(); - await persistHeartbeatRunProcessMetadata(db, run.id, { pid, processGroupId: null, startedAt: startedAt.toISOString() }); + await persistHeartbeatRunProcessMetadata(db, run.id, { pid, processGroupId: null, startedAt: startedAt.toISOString(), targetKind: "local" }); await advance(1_000); await vi.waitFor(async () => expect((await getRun(run.id)).processPid).toBeNull()); expect(await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, run.id), diff --git a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts index 1e259d81e63..997ed16f5c4 100644 --- a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts +++ b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts @@ -1,3 +1,5 @@ +import { persistHeartbeatRunProcessMetadata } from "../services/heartbeat/run-lifecycle.js"; +import { currentProcessSpaceHash } from "../services/controller-process-identity.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; @@ -394,6 +396,34 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { expect(second.cleared).toBe(0); }); + it.each([["local", true], ["remote", false]] as const)("stamps a %s process pid with its PID space", async (targetKind, local) => { + const { runningRunId } = await seed(); + await persistHeartbeatRunProcessMetadata(db, runningRunId, { pid: process.pid, processGroupId: null, + startedAt: new Date().toISOString(), targetKind }); + const [run] = await db.select({ processHostHash: heartbeatRuns.processHostHash }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, runningRunId)); + expect(run?.processHostHash).toBe(local ? currentProcessSpaceHash() : "remote"); + }); + + it.each([ + ["another host", "another-host-pid-space", false], + ["a sandbox", "remote", false], + ["an unreadable PID space", "unknown", false], + ["this PID space", "current", true], + ] as const)("judges a dead recorded pid only in its own PID space (%s)", async (_name, space, terminalized) => { + const { companyId, agentId, runningRunId } = await seed(); + await db.update(heartbeatRuns).set({ processPid: 2_000_000_000, + processHostHash: space === "current" ? currentProcessSpaceHash() : space }) + .where(eq(heartbeatRuns.id, runningRunId)); + await db.insert(issues).values({ id: randomUUID(), companyId, title: "Cross-host orphan", status: "in_progress", + priority: "high", assigneeAgentId: agentId, checkoutRunId: runningRunId, executionRunId: runningRunId, + executionLockedAt: new Date() }); + const result = await heartbeatService(db).sweepStaleIssueLocks(); + expect(result.terminalizedRunIds.includes(runningRunId)).toBe(terminalized); + const [run] = await db.select({ status: heartbeatRuns.status }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runningRunId)); + expect(run?.status).toBe(terminalized ? "interrupted" : "running"); + }); + it("terminalizes an orphaned running run whose process is gone, then clears the lock", async () => { const { companyId, agentId, runningRunId } = await seed(); // The run recorded a pid, but the process and its sandbox are gone. A pid diff --git a/server/src/services/explicit-native-continuation.test.ts b/server/src/services/explicit-native-continuation.test.ts index a1782ae298c..5e61f756765 100644 --- a/server/src/services/explicit-native-continuation.test.ts +++ b/server/src/services/explicit-native-continuation.test.ts @@ -1772,7 +1772,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes expect(await hasNativeLocalProcessStop(db, f.companyId, source.id)).toBe(true); expect(await hasNativeLocalProcessStop(db, randomUUID(), source.id)).toBe(false); expect(await admit(f, true)).toMatchObject({ previousRunId: source.id }); - await persistHeartbeatRunProcessMetadata(db, source.id, { pid: 999999999, processGroupId: null, startedAt: new Date().toISOString() }); + await persistHeartbeatRunProcessMetadata(db, source.id, { pid: 999999999, processGroupId: null, startedAt: new Date().toISOString(), targetKind: "local" }); await db.update(heartbeatRuns).set({ processPid: null }).where(eq(heartbeatRuns.id, source.id)); expect(await admit(f, true)).toBeNull(); expect(await db.transaction(tx => recordNativeLocalProcessStop(tx as unknown as typeof db, source))).toBe(true); diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 2ef0c635191..1a97b33057b 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -197,6 +197,7 @@ import { setHeartbeatRunRuntimeStatus, } from "../heartbeat-run-runtime-status.js"; import { readProcessStartedAt } from "../hot-restart.js"; +import { currentProcessSpaceHash } from "../controller-process-identity.js"; import type { createHeartbeatRunState } from "./run-state.js"; import type { createHeartbeatRunPreparation } from "./run-preparation.js"; @@ -578,7 +579,7 @@ function buildRunEventRuntimeProgress(input: { export async function persistHeartbeatRunProcessMetadata( db: Db, runId: string, - meta: { pid: number; processGroupId: number | null; startedAt: string; targetKind?: "local" | "remote" }, + meta: { pid: number; processGroupId: number | null; startedAt: string; targetKind: "local" | "remote" }, ) { // A sandbox PID belongs to a different host and may collide with a local PID. const observedStartedAt = meta.targetKind === "remote" ? null : await readProcessStartedAt(meta.pid).catch( @@ -592,6 +593,8 @@ export async function persistHeartbeatRunProcessMetadata( legacyProcessSettlement: null, processPid: meta.pid, processGroupId: meta.processGroupId, + // A sandbox PID never names a process in this PID space. + processHostHash: meta.targetKind === "remote" ? "remote" : currentProcessSpaceHash(), processStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt, @@ -2240,7 +2243,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl async function persistRunProcessMetadata( runId: string, - meta: { pid: number; processGroupId: number | null; startedAt: string; targetKind?: "local" | "remote" }, + meta: { pid: number; processGroupId: number | null; startedAt: string; targetKind: "local" | "remote" }, ) { return persistHeartbeatRunProcessMetadata(db, runId, meta); } diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index 37cb96940b8..d970aa4e2bb 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -444,6 +444,19 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expectCleanup(input); }); + it.each([["local", null], ["remote", { kind: "remote", transport: "sandbox", remoteCwd: "/workspace" }]] as const)( + "passes the spawn origin through the legacy adapter callback (%s)", async (expected, target) => { + const { input, adapter } = await fixture(); + if (target) input.workspace.executionTarget = target as typeof input.workspace.executionTarget; + adapter.execute.mockImplementationOnce(async ctx => { + await ctx.onSpawn?.({ pid: 4242, processGroupId: 4242, startedAt: new Date().toISOString() }); + return { exitCode: 0, signal: null, timedOut: false, provider: "test" }; + }); + await executeHeartbeatRuntime(db, input).catch(() => undefined); + expect(input.services.persistRunProcessMetadata).toHaveBeenCalledWith(expect.any(String), + expect.objectContaining({ pid: 4242, targetKind: expected })); + }); + it("preserves Stop metadata and marks no-work accounting without manufacturing a receipt", async () => { const { input, adapter, run } = await fixture(); await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { stoppedByUser: true } }).where(eq(heartbeatRuns.id, run.id)); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 6ffd6205b6a..88223c355cc 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -924,6 +924,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe ? meta.processGroupId : null, startedAt: meta.startedAt, + // A sandbox runner's spawn metadata names a sandbox PID. + targetKind: executionTarget?.kind ?? "local", }); }, authToken: authToken ?? undefined, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 30f3f03b832..7316605935c 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -108,6 +108,7 @@ import { isPidAlive, isProcessGroupAlive, } from "../local-service-supervisor.js"; +import { sameProcessSpace } from "../controller-process-identity.js"; import { redactSensitiveText } from "../../redaction.js"; import { isUniqueViolation } from "../../db-errors.js"; import { @@ -6180,7 +6181,12 @@ export function recoveryService( let processGone = false; const hasLiveExecution = deps.liveRunExecutions?.has(run.id) ?? runningProcesses.has(run.id); - if (!hasLiveExecution) { + // A recorded PID is judged only in the PID space that recorded it: a dead + // local PID never proves that a process on another host, in another + // namespace, or in a sandbox ("remote") is gone. Rows saved before the + // space was recorded keep the previous local proof for one release. + const processSpaceJudgeable = run.processHostHash == null || sameProcessSpace(run.processHostHash); + if (!hasLiveExecution && processSpaceJudgeable) { if (typeof pid === "number" || typeof processGroupId === "number") { const processAlive = (typeof pid === "number" && isPidAlive(pid)) ||