diff --git a/packages/db/src/migrations/0441_lucky_chameleon.sql b/packages/db/src/migrations/0441_lucky_chameleon.sql new file mode 100644 index 00000000000..7c1f0f4734e --- /dev/null +++ b/packages/db/src/migrations/0441_lucky_chameleon.sql @@ -0,0 +1 @@ +ALTER TABLE "native_run_finalizations" ADD COLUMN "controller_host_hash" text; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0436_snapshot.json b/packages/db/src/migrations/meta/0441_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0436_snapshot.json rename to packages/db/src/migrations/meta/0441_snapshot.json index f89861310b7..6e0bbf97914 100644 --- a/packages/db/src/migrations/meta/0436_snapshot.json +++ b/packages/db/src/migrations/meta/0441_snapshot.json @@ -1,6 +1,6 @@ { - "id": "a80d63fe-5111-49e8-a39a-7a5589b3b31a", - "prevId": "dedfce21-7ff9-4079-b5da-4a74a856a251", + "id": "a2ec96c6-3cf1-4c51-8ee8-b35f090bfd42", + "prevId": "0c990bdc-6df0-44ca-8f21-28124f7b39e7", "version": "7", "dialect": "postgresql", "tables": { @@ -28984,6 +28984,22 @@ "method": "btree", "with": {} }, + "environments_managed_plugin_environment_idx": { + "name": "environments_managed_plugin_environment_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'managedPluginEnvironment')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "jsonb_typeof(\"environments\".\"metadata\" -> 'managedPluginEnvironment') = 'string'", + "concurrently": false, + "method": "btree", + "with": {} + }, "environments_name_idx": { "name": "environments_name_idx", "columns": [ @@ -34864,6 +34880,12 @@ "primaryKey": false, "notNull": false }, + "legacy_process_settlement": { + "name": "legacy_process_settlement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "last_output_at": { "name": "last_output_at", "type": "timestamp with time zone", @@ -44441,6 +44463,256 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.native_provider_artifact_references": { + "name": "native_provider_artifact_references", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifact_references_artifact_idx": { + "name": "native_provider_artifact_references_artifact_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_run_idx": { + "name": "native_provider_artifact_references_run_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_backup_idx": { + "name": "native_provider_artifact_references_backup_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk": { + "name": "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_provider_artifacts", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "native_provider_artifact_references_backup_id_native_harness_backups_id_fk": { + "name": "native_provider_artifact_references_backup_id_native_harness_backups_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_harness_backups", + "columnsFrom": [ + "backup_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.native_provider_artifacts": { + "name": "native_provider_artifacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "closure_digest": { + "name": "closure_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "upload_token": { + "name": "upload_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "descriptor": { + "name": "descriptor", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reference": { + "name": "reference", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifacts_company_digest_idx": { + "name": "native_provider_artifacts_company_digest_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closure_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifacts\".\"status\" = 'ready'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.native_run_finalizations": { "name": "native_run_finalizations", "schema": "", @@ -44500,6 +44772,12 @@ "primaryKey": false, "notNull": false }, + "controller_host_hash": { + "name": "controller_host_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, "controller_process_started_at": { "name": "controller_process_started_at", "type": "timestamp with time zone", @@ -64059,6 +64337,139 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.mcp_oauth_client_changes": { + "name": "mcp_oauth_client_changes", + "schema": "", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_client_changes_revision_idx": { + "name": "mcp_client_changes_revision_idx", + "columns": [ + { + "expression": "revision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_client_change_revision": { + "name": "mcp_client_change_revision", + "value": "\"mcp_oauth_client_changes\".\"revision\" >= 1" + }, + "mcp_client_change_operation": { + "name": "mcp_client_change_operation", + "value": "\"mcp_oauth_client_changes\".\"operation\" IN ('upsert','delete')" + } + }, + "isRLSEnabled": false + }, + "public.mcp_oauth_client_mirror_state": { + "name": "mcp_oauth_client_mirror_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "remote_responsibility": { + "name": "remote_responsibility", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "device_cutover_stack_id": { + "name": "device_cutover_stack_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_cutover_at": { + "name": "device_cutover_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "receipt": { + "name": "receipt", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_client_mirror_singleton": { + "name": "mcp_client_mirror_singleton", + "value": "\"mcp_oauth_client_mirror_state\".\"id\" = 'public'" + }, + "mcp_client_mirror_revision": { + "name": "mcp_client_mirror_revision", + "value": "\"mcp_oauth_client_mirror_state\".\"revision\" >= 0" + }, + "mcp_device_cutover_identity": { + "name": "mcp_device_cutover_identity", + "value": "(\"mcp_oauth_client_mirror_state\".\"device_cutover_at\" IS NULL AND \"mcp_oauth_client_mirror_state\".\"device_cutover_stack_id\" IS NULL) OR (\"mcp_oauth_client_mirror_state\".\"device_cutover_at\" IS NOT NULL AND \"mcp_oauth_client_mirror_state\".\"device_cutover_stack_id\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, "public.mcp_oauth_clients": { "name": "mcp_oauth_clients", "schema": "", @@ -64127,6 +64538,13 @@ "primaryKey": false, "notNull": true }, + "ingress_owner": { + "name": "ingress_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'tenant'" + }, "device_code_hash": { "name": "device_code_hash", "type": "text", @@ -64282,7 +64700,12 @@ "compositePrimaryKeys": {}, "uniqueConstraints": {}, "policies": {}, - "checkConstraints": {}, + "checkConstraints": { + "mcp_device_ingress_owner": { + "name": "mcp_device_ingress_owner", + "value": "\"mcp_oauth_device_requests\".\"ingress_owner\" IN ('tenant','cloud')" + } + }, "isRLSEnabled": false }, "public.mcp_oauth_grants": { @@ -64345,6 +64768,19 @@ "primaryKey": false, "notNull": false }, + "revocation_revision": { + "name": "revocation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "revocation_index_receipt": { + "name": "revocation_index_receipt", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -64675,6 +65111,13 @@ "primaryKey": false, "notNull": false }, + "revocation_revision": { + "name": "revocation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, "expires_at": { "name": "expires_at", "type": "timestamp with time zone", @@ -64720,6 +65163,33 @@ "method": "btree", "with": {} }, + "mcp_oauth_tokens_revocation_revision_idx": { + "name": "mcp_oauth_tokens_revocation_revision_idx", + "columns": [ + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "revocation_revision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "mcp_oauth_tokens_expiry_idx": { "name": "mcp_oauth_tokens_expiry_idx", "columns": [ diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 90aa9291d5a..08d4023c1f1 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3074,6 +3074,13 @@ "when": 1794035686391, "tag": "0440_flippant_the_leader", "breakpoints": true + }, + { + "idx": 441, + "version": "7", + "when": 1794069983192, + "tag": "0441_lucky_chameleon", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/native_run_finalizations.ts b/packages/db/src/schema/native_run_finalizations.ts index aaadc6fc97c..4f195f984dc 100644 --- a/packages/db/src/schema/native_run_finalizations.ts +++ b/packages/db/src/schema/native_run_finalizations.ts @@ -29,6 +29,9 @@ export const nativeRunFinalizations = pgTable( leaseExpiresAt: timestamp("lease_expires_at", { withTimezone: true }), controllerBootId: text("controller_boot_id"), controllerPid: integer("controller_pid"), + // PID space of controllerPid (controller-process-identity hostHash). Null + // on rows saved before it existed. + controllerHostHash: text("controller_host_hash"), controllerProcessStartedAt: timestamp("controller_process_started_at", { withTimezone: true, }), diff --git a/server/src/__tests__/workspace-runtime-control-recovery.test.ts b/server/src/__tests__/workspace-runtime-control-recovery.test.ts index 52125e4af84..62d3a6fce05 100644 --- a/server/src/__tests__/workspace-runtime-control-recovery.test.ts +++ b/server/src/__tests__/workspace-runtime-control-recovery.test.ts @@ -19,6 +19,7 @@ import { workspaceOperationService, workspaceRuntimeControlOwnerIdForTests, } from "../services/workspace-operations.js"; +import { currentProcessSpaceHash } from "../services/controller-process-identity.js"; import { getWorkspaceOperationLogStore } from "../services/workspace-operation-log-store.js"; import { getEmbeddedPostgresTestSupport, @@ -124,7 +125,8 @@ describeEmbeddedPostgres("managed runtime-control operation recovery", () => { async function seedRunningStart(input: { companyId: string; executionWorkspaceId: string; - owner?: { ownerId: string; pid: number; heartbeatAt: Date }; + /** hostHash defaults to this process's PID space; null omits it (a legacy stamp). */ + owner?: { ownerId: string; pid: number; heartbeatAt: Date; hostHash?: string | null }; withLog?: boolean; startedAt?: Date; /** @@ -157,6 +159,7 @@ describeEmbeddedPostgres("managed runtime-control operation recovery", () => { runtimeControlOwner: { ownerId: input.owner.ownerId, pid: input.owner.pid, + ...(input.owner.hostHash === null ? {} : { hostHash: input.owner.hostHash ?? currentProcessSpaceHash() }), action: "start", heartbeatAt: input.owner.heartbeatAt.toISOString(), }, @@ -204,6 +207,25 @@ describeEmbeddedPostgres("managed runtime-control operation recovery", () => { ).resolves.toBeUndefined(); }); + it.each([ + ["another host", "another-host-pid-space"], + ["an unreadable PID space", "unknown"], + ["a stamp with no recorded PID space", null], + ] as const)("never treats a dead local pid as proof for an owner on %s", async (_name, hostHash) => { + const { executionWorkspaceId } = await seedWorkspace().then(async (seeded) => { + await seedRunningStart({ ...seeded, + owner: { ownerId: randomUUID(), pid: 4_194_303, heartbeatAt: new Date(), hostHash } }); + return seeded; + }); + const service = workspaceOperationService(db); + // Unprovable: inside the staleness window the operation stays live. + await expect(service.reconcileStaleRuntimeControlOperations(executionWorkspaceId)).resolves.toEqual({ + reconciled: 0, operationIds: [] }); + // Its existing TTL still applies once the owner's heartbeat lapses. + await expect(service.reconcileStaleRuntimeControlOperations(executionWorkspaceId, { + now: new Date(Date.now() + 60 * 60_000) })).resolves.toMatchObject({ reconciled: 1 }); + }); + it("refuses to steal a start whose owning process is still alive inside the staleness window", async () => { const { companyId, executionWorkspaceId } = await seedWorkspace(); const service = workspaceOperationService(db); diff --git a/server/src/services/controller-process-identity.ts b/server/src/services/controller-process-identity.ts index 1a537a1eb0b..6065aa41408 100644 --- a/server/src/services/controller-process-identity.ts +++ b/server/src/services/controller-process-identity.ts @@ -142,3 +142,18 @@ export async function controllerProcessExited(controller: ControllerProcess): Pr const observed = await observedProcessStartedAt(controller.pid); return observed !== null && processStartsDefinitelyDiffer(controller.processStartedAt, new Date(observed)); } + +/** This process's PID-space hash. An unreadable space is stored as + * "unknown", which `sameProcessSpace` never accepts. */ +export function currentProcessSpaceHash(): string { + return hostHash; +} + +/** + * Whether a PID recorded under `recorded` can be judged in this process. + * True only for the same readable PID space. A different or unreadable space + * is never proof: a dead PID here says nothing about a process elsewhere. + */ +export function sameProcessSpace(recorded: string | null | undefined): boolean { + return typeof recorded === "string" && hostHash !== UNKNOWN_PROCESS_SPACE && recorded === hostHash; +} diff --git a/server/src/services/native-runtime/native-dead-controller-result-lease.ts b/server/src/services/native-runtime/native-dead-controller-result-lease.ts index 7258870e2c1..fcf922eb024 100644 --- a/server/src/services/native-runtime/native-dead-controller-result-lease.ts +++ b/server/src/services/native-runtime/native-dead-controller-result-lease.ts @@ -35,6 +35,7 @@ export async function releaseDeadControllerResultLeases(db: Db, input: { leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, controllerBootId: nativeRunFinalizations.controllerBootId, controllerPid: nativeRunFinalizations.controllerPid, + controllerHostHash: nativeRunFinalizations.controllerHostHash, controllerProcessStartedAt: nativeRunFinalizations.controllerProcessStartedAt, }).from(nativeRunFinalizations) .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index 52d8d15c9b7..eaa8708e510 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -1,3 +1,4 @@ +import { currentProcessSpaceHash } from "../controller-process-identity.js"; import { describe, expect, it, vi } from "vitest"; import { @@ -269,6 +270,25 @@ describe("native controller takeover fencing", () => { }); }); + it.each([ + ["another host", "another-host-pid-space"], + ["an unreadable PID space", "unknown"], + ])("never treats a dead or recycled local PID as proof for a controller on %s", async (_name, controllerHostHash) => { + for (const probe of [{ isProcessAlive: () => false }, { isProcessAlive: () => true, + readProcessStartedAt: async () => new Date("2026-09-04T11:30:00.000Z") }]) { + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash }, now, ...probe })) + .resolves.toEqual({ allowed: false, reason: "controller_host_unprovable" }); + } + }); + + it("proves a controller gone only in its own recorded PID space", async () => { + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: currentProcessSpaceHash() }, + now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); + // A row saved before the PID space was recorded keeps the previous local proof for one release. + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: null }, + now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); + }); + it.each([true, false])("requires a stopped idle controller even after its run lease was released: alive=%s", async alive => { await expect(evaluateNativeControllerTakeover({ owner: owner({ leaseOwner: null, leaseExpiresAt: null }), now, requireStoppedController: true, diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 54c4087d299..98d8444ee24 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -35,7 +35,7 @@ import { } from "@paperclipai/db"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { readProcessStartedAt } from "../hot-restart.js"; -import { currentControllerBootId, currentProcessStartedAtMs, processStartsDefinitelyDiffer } from "../controller-process-identity.js"; +import { currentControllerBootId, currentProcessSpaceHash, currentProcessStartedAtMs, processStartsDefinitelyDiffer, sameProcessSpace } from "../controller-process-identity.js"; import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; import { isUniqueViolation } from "../../db-errors.js"; @@ -54,6 +54,9 @@ export type NativeControllerIdentity = { bootId: string; pid: number; processStartedAt: Date; + /** PID space of `pid`; "unknown" when this host's space cannot be read, + * which no later probe accepts. Optional for identities built in tests. */ + hostHash?: string; }; export type NativeRestartKind = "hot" | "hard" | "graceful"; @@ -334,6 +337,7 @@ export async function currentNativeControllerIdentity(): Promise; + > & { + /** Absent for owners without a stored PID space (receipts, rows saved + * before the column existed): those keep the previous local proof. */ + controllerHostHash?: string | null; + }; now: Date; coordinatedPreviousController?: { pid: number; @@ -396,6 +404,12 @@ export async function evaluateNativeControllerTakeover(input: { : "live_legacy_lease_without_controller_identity", }; } + // A PID can only be judged in the PID space that recorded it. A controller + // on another host or namespace (or an unreadable local space) is never + // proven gone by a local probe; its lease or fence decides instead. + if (owner.controllerHostHash && !sameProcessSpace(owner.controllerHostHash)) { + return { allowed: false, reason: "controller_host_unprovable" }; + } if (!isAlive(priorPid)) { return { allowed: true, @@ -1522,6 +1536,7 @@ export async function claimNativeRestartRecoveries(input: { leaseExpiresAt: sql`clock_timestamp() + ${durableAuthority ? REMOTE_CONTROLLER_LEASE_MS : 20 * 60_000} * interval '1 millisecond'`, controllerBootId: controller.bootId, controllerPid: controller.pid, + controllerHostHash: controller.hostHash ?? null, controllerProcessStartedAt: controller.processStartedAt, controllerGeneration: generation, recoveryState: diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 4ee6b2a0305..4e18dae27f2 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -9831,6 +9831,7 @@ async function executePaperclipNativeSessionWithinScope( leaseExpiresAt: sql`clock_timestamp() + ${executionLeaseTtlMs} * interval '1 millisecond'`, controllerBootId: controller.bootId, controllerPid: controller.pid, + controllerHostHash: controller.hostHash ?? null, controllerProcessStartedAt: controller.processStartedAt, controllerGeneration: nextControllerGeneration, failureCode: null, diff --git a/server/src/services/workspace-operations.ts b/server/src/services/workspace-operations.ts index acf39177275..ab2aa012b7f 100644 --- a/server/src/services/workspace-operations.ts +++ b/server/src/services/workspace-operations.ts @@ -11,6 +11,7 @@ import { asc, desc, eq, gte, inArray, isNull, lt, or, and } from "drizzle-orm"; import { conflict, notFound } from "../errors.js"; import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js"; import { instanceSettingsService } from "./instance-settings.js"; +import { currentProcessSpaceHash, processExists, sameProcessSpace } from "./controller-process-identity.js"; import { getWorkspaceOperationLogStore } from "./workspace-operation-log-store.js"; type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect; @@ -71,6 +72,8 @@ type RuntimeControlOwnerStamp = { pid: number; action: string; heartbeatAt: string; + /** PID space of `pid`. Absent on stamps written before it was recorded. */ + hostHash?: string; }; function readRuntimeControlAction(metadata: unknown): string | null { @@ -91,6 +94,7 @@ function readRuntimeControlOwner(metadata: unknown): RuntimeControlOwnerStamp | pid: typeof record.pid === "number" ? record.pid : 0, action: typeof record.action === "string" ? record.action : "start", heartbeatAt: typeof record.heartbeatAt === "string" ? record.heartbeatAt : new Date(0).toISOString(), + ...(typeof record.hostHash === "string" ? { hostHash: record.hostHash } : {}), }; } @@ -118,20 +122,17 @@ export function isNativeWorkspaceFinalizationOperationActive(scope: NativeFinali } /** - * Whether the process that stamped an operation is still alive on this host. A dead owner can - * never heartbeat again, so its operation is recoverable immediately instead of after the - * staleness window — which is what makes recovery after a crash or restart prompt. Signal 0 - * only probes; it never touches the process. + * Whether the process that stamped an operation has provably exited. A dead owner can never + * heartbeat again, so its operation is recoverable immediately instead of after the staleness + * window — which is what makes recovery after a crash or restart prompt. The PID is judged only + * in the PID space that recorded it: an owner on another host or namespace, an unreadable + * space, or a stamp without a recorded space is never proven gone here and waits out the + * staleness window instead. Signal 0 only probes; EPERM means the process exists. */ -function ownerProcessIsAlive(pid: number): boolean { - if (!Number.isInteger(pid) || pid <= 0) return false; - try { - process.kill(pid, 0); - return true; - } catch (error) { - // EPERM means the pid exists but belongs to another user: treat it as alive. - return (error as NodeJS.ErrnoException)?.code === "EPERM"; - } +function ownerProcessProvenGone(owner: RuntimeControlOwnerStamp): boolean { + if (!sameProcessSpace(owner.hostHash)) return false; + if (!Number.isInteger(owner.pid) || owner.pid <= 0) return false; + return !processExists(owner.pid); } /** @@ -336,7 +337,7 @@ export function workspaceOperationService(db: Db) { const ownedHereAndLive = ownedByThisProcess && liveRuntimeControlOperationIds.has(candidate.id); // A previous server process that no longer exists cannot heartbeat again, so its // operations are recoverable at once instead of after the staleness window. - const ownerProcessGone = owner !== null && !ownedByThisProcess && !ownerProcessIsAlive(owner.pid); + const ownerProcessGone = owner !== null && !ownedByThisProcess && ownerProcessProvenGone(owner); if (ownedHereAndLive) { // This process is driving it right now; its own time budget is what terminalizes it. @@ -543,6 +544,7 @@ export function workspaceOperationService(db: Db) { runtimeControlOwner: { ownerId: RUNTIME_CONTROL_OWNER_ID, pid: process.pid, + hostHash: currentProcessSpaceHash(), action: runtimeControlAction, heartbeatAt: startedAt.toISOString(), } satisfies RuntimeControlOwnerStamp, @@ -594,6 +596,7 @@ export function workspaceOperationService(db: Db) { runtimeControlOwner: { ownerId: RUNTIME_CONTROL_OWNER_ID, pid: process.pid, + hostHash: currentProcessSpaceHash(), action: runtimeControlAction, heartbeatAt: heartbeatAt.toISOString(), } satisfies RuntimeControlOwnerStamp,