From 68db7ed9e36954473af82cce7b29fcb36c0805cd Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:14:48 -0600 Subject: [PATCH] fix(recovery): never accept a dead local pid as proof for an owner on another host (#19868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - After a restart, recovery decides that an in-progress record's owner is gone and takes it over > - Two of those proofs use `kill(pid, 0)` on the local host. A pid that is dead here says nothing about a process on another host or in another PID namespace > - So a controller or runtime-control owner that is still alive elsewhere can be taken over while it still runs > - This pull request records the owner's PID space (the existing `controller-process-identity` host hash) and judges a pid only in that same space > - The benefit is that a different host is never proof. Those records follow their existing lease or TTL instead ## Linked Issues or Issue Description **Bug description** The Phase 0 ownership inventory (finding 2) found three proofs that accept "the pid is dead here" as proof for an owner on another host: `evaluateNativeControllerTakeover` (`native_run_finalizations` has no host column), the runtime-control sweep (`workspace-operations.ts`), and the orphan reaper `terminalizeOrphanedRunningRun`. Only `controllerProcessExited` checked the host. **Steps to reproduce** 1. Run two control-plane processes on different hosts, or in different PID namespaces, against one database. 2. Let host A own a native finalization lease or a runtime-control operation. 3. On host B, run restart recovery. If A's pid number is not alive on B, B takes over the live owner. **Expected behavior** An owner whose PID space differs from this process's, or cannot be read, is unprovable. Its record waits for its lease, fence, or TTL. Refs #19846 ## What Changed - Migration `0441`: add nullable `native_run_finalizations.controller_host_hash`. - `controller-process-identity.ts`: export `currentProcessSpaceHash()` and `sameProcessSpace()`. An unreadable space is stored as `"unknown"`, which never matches. - `native-restart-recovery.ts`: `currentNativeControllerIdentity` carries `hostHash`, which both coordinator claim writes (`native-restart-recovery.ts`, `native-session-executor.ts`) store. `evaluateNativeControllerTakeover` returns `controller_host_unprovable` before any pid probe when the recorded space is not this one. `native-dead-controller-result-lease.ts` selects the column. - `workspace-operations.ts`: runtime-control owner stamps record `hostHash`. `ownerProcessProvenGone` requires the same PID space (EPERM = alive), otherwise the existing staleness TTL applies. - Not in this PR: the orphan reaper `terminalizeOrphanedRunningRun`. `heartbeat_runs` has no host for `process_pid`, and its writers are spread across legacy and native paths. It follows as a separate PR with its own column. ## Verification - `npx vitest run src/services/native-runtime/native-restart-recovery.test.ts src/__tests__/workspace-runtime-control-recovery.test.ts`: 55 passed. New cases cover another host and an unreadable space (dead and recycled pid) for takeover, plus another host, unreadable space, and legacy stamp for the sweep. The sweep cases also confirm that the TTL still reclaims after the heartbeat lapses. - Red before the fix: the 5 new cases fail on the old code. Green after. - `native-controller-recovery.integration.test.ts`, `native-failed-run-owner-release.test.ts`, `native-startup-cleanup-recovery.test.ts`: 71 passed. - `pnpm db:generate` produced `0441_lucky_chameleon.sql` (one `ADD COLUMN`) and pruned to five snapshots. `pnpm --filter @paperclipai/db check:migrations` passed. Server `tsc --noEmit` shows no errors in the changed files. ## Risks - Previous-release story: `native_run_finalizations` rows saved before this column have `controller_host_hash = null`. They keep the previous local dead-pid proof for this release. Making them unprovable would leave every native run in flight at the upgrade in `awaiting_evidence` with no TTL, because local takeovers have no lease fallback. Rows written by this release onward are strict. Runtime-control stamps without `hostHash` are strict now, because that path has a staleness TTL. - Owners built from other receipts (workspace finalization owner, workspace stop proof) pass no host and keep the previous behavior. Track 2's shared `OwnerStamp` should absorb them. - A new row on a host that really moved (redeploy onto a new machine) waits for its lease or an operator instead of being taken over by pid. That is the intended "different host is never proof" rule. - No new I/O before `turn.start`. The hash is computed once at module load. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../src/migrations/0441_lucky_chameleon.sql | 1 + ...{0436_snapshot.json => 0441_snapshot.json} | 476 +++++++++++++++++- packages/db/src/migrations/meta/_journal.json | 7 + .../db/src/schema/native_run_finalizations.ts | 3 + ...workspace-runtime-control-recovery.test.ts | 24 +- .../services/controller-process-identity.ts | 15 + .../native-dead-controller-result-lease.ts | 1 + .../native-restart-recovery.test.ts | 20 + .../native-runtime/native-restart-recovery.ts | 19 +- .../native-runtime/native-session-executor.ts | 1 + server/src/services/workspace-operations.ts | 31 +- 11 files changed, 578 insertions(+), 20 deletions(-) create mode 100644 packages/db/src/migrations/0441_lucky_chameleon.sql rename packages/db/src/migrations/meta/{0436_snapshot.json => 0441_snapshot.json} (99%) diff --git a/packages/db/src/migrations/0441_lucky_chameleon.sql b/packages/db/src/migrations/0441_lucky_chameleon.sql new file mode 100644 index 00000000000..7c1f0f4734e --- /dev/null +++ b/packages/db/src/migrations/0441_lucky_chameleon.sql @@ -0,0 +1 @@ +ALTER TABLE "native_run_finalizations" ADD COLUMN "controller_host_hash" text; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0436_snapshot.json b/packages/db/src/migrations/meta/0441_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0436_snapshot.json rename to packages/db/src/migrations/meta/0441_snapshot.json index f89861310b7..6e0bbf97914 100644 --- a/packages/db/src/migrations/meta/0436_snapshot.json +++ b/packages/db/src/migrations/meta/0441_snapshot.json @@ -1,6 +1,6 @@ { - "id": "a80d63fe-5111-49e8-a39a-7a5589b3b31a", - "prevId": "dedfce21-7ff9-4079-b5da-4a74a856a251", + "id": "a2ec96c6-3cf1-4c51-8ee8-b35f090bfd42", + "prevId": "0c990bdc-6df0-44ca-8f21-28124f7b39e7", "version": "7", "dialect": "postgresql", "tables": { @@ -28984,6 +28984,22 @@ "method": "btree", "with": {} }, + "environments_managed_plugin_environment_idx": { + "name": "environments_managed_plugin_environment_idx", + "columns": [ + { + "expression": "(\"metadata\" ->> 'managedPluginEnvironment')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "jsonb_typeof(\"environments\".\"metadata\" -> 'managedPluginEnvironment') = 'string'", + "concurrently": false, + "method": "btree", + "with": {} + }, "environments_name_idx": { "name": "environments_name_idx", "columns": [ @@ -34864,6 +34880,12 @@ "primaryKey": false, "notNull": false }, + "legacy_process_settlement": { + "name": "legacy_process_settlement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "last_output_at": { "name": "last_output_at", "type": "timestamp with time zone", @@ -44441,6 +44463,256 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.native_provider_artifact_references": { + "name": "native_provider_artifact_references", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "backup_id": { + "name": "backup_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifact_references_artifact_idx": { + "name": "native_provider_artifact_references_artifact_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_run_idx": { + "name": "native_provider_artifact_references_run_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "native_provider_artifact_references_backup_idx": { + "name": "native_provider_artifact_references_backup_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "artifact_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "backup_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifact_references\".\"backup_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk": { + "name": "native_provider_artifact_references_artifact_id_native_provider_artifacts_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_provider_artifacts", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "native_provider_artifact_references_backup_id_native_harness_backups_id_fk": { + "name": "native_provider_artifact_references_backup_id_native_harness_backups_id_fk", + "tableFrom": "native_provider_artifact_references", + "tableTo": "native_harness_backups", + "columnsFrom": [ + "backup_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.native_provider_artifacts": { + "name": "native_provider_artifacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "closure_digest": { + "name": "closure_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "upload_token": { + "name": "upload_token", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "descriptor": { + "name": "descriptor", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reference": { + "name": "reference", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "native_provider_artifacts_company_digest_idx": { + "name": "native_provider_artifacts_company_digest_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "closure_digest", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"native_provider_artifacts\".\"status\" = 'ready'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.native_run_finalizations": { "name": "native_run_finalizations", "schema": "", @@ -44500,6 +44772,12 @@ "primaryKey": false, "notNull": false }, + "controller_host_hash": { + "name": "controller_host_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, "controller_process_started_at": { "name": "controller_process_started_at", "type": "timestamp with time zone", @@ -64059,6 +64337,139 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.mcp_oauth_client_changes": { + "name": "mcp_oauth_client_changes", + "schema": "", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_client_changes_revision_idx": { + "name": "mcp_client_changes_revision_idx", + "columns": [ + { + "expression": "revision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "client_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_client_change_revision": { + "name": "mcp_client_change_revision", + "value": "\"mcp_oauth_client_changes\".\"revision\" >= 1" + }, + "mcp_client_change_operation": { + "name": "mcp_client_change_operation", + "value": "\"mcp_oauth_client_changes\".\"operation\" IN ('upsert','delete')" + } + }, + "isRLSEnabled": false + }, + "public.mcp_oauth_client_mirror_state": { + "name": "mcp_oauth_client_mirror_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "remote_responsibility": { + "name": "remote_responsibility", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "device_cutover_stack_id": { + "name": "device_cutover_stack_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_cutover_at": { + "name": "device_cutover_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "receipt": { + "name": "receipt", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_client_mirror_singleton": { + "name": "mcp_client_mirror_singleton", + "value": "\"mcp_oauth_client_mirror_state\".\"id\" = 'public'" + }, + "mcp_client_mirror_revision": { + "name": "mcp_client_mirror_revision", + "value": "\"mcp_oauth_client_mirror_state\".\"revision\" >= 0" + }, + "mcp_device_cutover_identity": { + "name": "mcp_device_cutover_identity", + "value": "(\"mcp_oauth_client_mirror_state\".\"device_cutover_at\" IS NULL AND \"mcp_oauth_client_mirror_state\".\"device_cutover_stack_id\" IS NULL) OR (\"mcp_oauth_client_mirror_state\".\"device_cutover_at\" IS NOT NULL AND \"mcp_oauth_client_mirror_state\".\"device_cutover_stack_id\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, "public.mcp_oauth_clients": { "name": "mcp_oauth_clients", "schema": "", @@ -64127,6 +64538,13 @@ "primaryKey": false, "notNull": true }, + "ingress_owner": { + "name": "ingress_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'tenant'" + }, "device_code_hash": { "name": "device_code_hash", "type": "text", @@ -64282,7 +64700,12 @@ "compositePrimaryKeys": {}, "uniqueConstraints": {}, "policies": {}, - "checkConstraints": {}, + "checkConstraints": { + "mcp_device_ingress_owner": { + "name": "mcp_device_ingress_owner", + "value": "\"mcp_oauth_device_requests\".\"ingress_owner\" IN ('tenant','cloud')" + } + }, "isRLSEnabled": false }, "public.mcp_oauth_grants": { @@ -64345,6 +64768,19 @@ "primaryKey": false, "notNull": false }, + "revocation_revision": { + "name": "revocation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "revocation_index_receipt": { + "name": "revocation_index_receipt", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -64675,6 +65111,13 @@ "primaryKey": false, "notNull": false }, + "revocation_revision": { + "name": "revocation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, "expires_at": { "name": "expires_at", "type": "timestamp with time zone", @@ -64720,6 +65163,33 @@ "method": "btree", "with": {} }, + "mcp_oauth_tokens_revocation_revision_idx": { + "name": "mcp_oauth_tokens_revocation_revision_idx", + "columns": [ + { + "expression": "grant_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "revocation_revision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "mcp_oauth_tokens_expiry_idx": { "name": "mcp_oauth_tokens_expiry_idx", "columns": [ diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 90aa9291d5a..08d4023c1f1 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3074,6 +3074,13 @@ "when": 1794035686391, "tag": "0440_flippant_the_leader", "breakpoints": true + }, + { + "idx": 441, + "version": "7", + "when": 1794069983192, + "tag": "0441_lucky_chameleon", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/native_run_finalizations.ts b/packages/db/src/schema/native_run_finalizations.ts index aaadc6fc97c..4f195f984dc 100644 --- a/packages/db/src/schema/native_run_finalizations.ts +++ b/packages/db/src/schema/native_run_finalizations.ts @@ -29,6 +29,9 @@ export const nativeRunFinalizations = pgTable( leaseExpiresAt: timestamp("lease_expires_at", { withTimezone: true }), controllerBootId: text("controller_boot_id"), controllerPid: integer("controller_pid"), + // PID space of controllerPid (controller-process-identity hostHash). Null + // on rows saved before it existed. + controllerHostHash: text("controller_host_hash"), controllerProcessStartedAt: timestamp("controller_process_started_at", { withTimezone: true, }), diff --git a/server/src/__tests__/workspace-runtime-control-recovery.test.ts b/server/src/__tests__/workspace-runtime-control-recovery.test.ts index 52125e4af84..62d3a6fce05 100644 --- a/server/src/__tests__/workspace-runtime-control-recovery.test.ts +++ b/server/src/__tests__/workspace-runtime-control-recovery.test.ts @@ -19,6 +19,7 @@ import { workspaceOperationService, workspaceRuntimeControlOwnerIdForTests, } from "../services/workspace-operations.js"; +import { currentProcessSpaceHash } from "../services/controller-process-identity.js"; import { getWorkspaceOperationLogStore } from "../services/workspace-operation-log-store.js"; import { getEmbeddedPostgresTestSupport, @@ -124,7 +125,8 @@ describeEmbeddedPostgres("managed runtime-control operation recovery", () => { async function seedRunningStart(input: { companyId: string; executionWorkspaceId: string; - owner?: { ownerId: string; pid: number; heartbeatAt: Date }; + /** hostHash defaults to this process's PID space; null omits it (a legacy stamp). */ + owner?: { ownerId: string; pid: number; heartbeatAt: Date; hostHash?: string | null }; withLog?: boolean; startedAt?: Date; /** @@ -157,6 +159,7 @@ describeEmbeddedPostgres("managed runtime-control operation recovery", () => { runtimeControlOwner: { ownerId: input.owner.ownerId, pid: input.owner.pid, + ...(input.owner.hostHash === null ? {} : { hostHash: input.owner.hostHash ?? currentProcessSpaceHash() }), action: "start", heartbeatAt: input.owner.heartbeatAt.toISOString(), }, @@ -204,6 +207,25 @@ describeEmbeddedPostgres("managed runtime-control operation recovery", () => { ).resolves.toBeUndefined(); }); + it.each([ + ["another host", "another-host-pid-space"], + ["an unreadable PID space", "unknown"], + ["a stamp with no recorded PID space", null], + ] as const)("never treats a dead local pid as proof for an owner on %s", async (_name, hostHash) => { + const { executionWorkspaceId } = await seedWorkspace().then(async (seeded) => { + await seedRunningStart({ ...seeded, + owner: { ownerId: randomUUID(), pid: 4_194_303, heartbeatAt: new Date(), hostHash } }); + return seeded; + }); + const service = workspaceOperationService(db); + // Unprovable: inside the staleness window the operation stays live. + await expect(service.reconcileStaleRuntimeControlOperations(executionWorkspaceId)).resolves.toEqual({ + reconciled: 0, operationIds: [] }); + // Its existing TTL still applies once the owner's heartbeat lapses. + await expect(service.reconcileStaleRuntimeControlOperations(executionWorkspaceId, { + now: new Date(Date.now() + 60 * 60_000) })).resolves.toMatchObject({ reconciled: 1 }); + }); + it("refuses to steal a start whose owning process is still alive inside the staleness window", async () => { const { companyId, executionWorkspaceId } = await seedWorkspace(); const service = workspaceOperationService(db); diff --git a/server/src/services/controller-process-identity.ts b/server/src/services/controller-process-identity.ts index 1a537a1eb0b..6065aa41408 100644 --- a/server/src/services/controller-process-identity.ts +++ b/server/src/services/controller-process-identity.ts @@ -142,3 +142,18 @@ export async function controllerProcessExited(controller: ControllerProcess): Pr const observed = await observedProcessStartedAt(controller.pid); return observed !== null && processStartsDefinitelyDiffer(controller.processStartedAt, new Date(observed)); } + +/** This process's PID-space hash. An unreadable space is stored as + * "unknown", which `sameProcessSpace` never accepts. */ +export function currentProcessSpaceHash(): string { + return hostHash; +} + +/** + * Whether a PID recorded under `recorded` can be judged in this process. + * True only for the same readable PID space. A different or unreadable space + * is never proof: a dead PID here says nothing about a process elsewhere. + */ +export function sameProcessSpace(recorded: string | null | undefined): boolean { + return typeof recorded === "string" && hostHash !== UNKNOWN_PROCESS_SPACE && recorded === hostHash; +} diff --git a/server/src/services/native-runtime/native-dead-controller-result-lease.ts b/server/src/services/native-runtime/native-dead-controller-result-lease.ts index 7258870e2c1..fcf922eb024 100644 --- a/server/src/services/native-runtime/native-dead-controller-result-lease.ts +++ b/server/src/services/native-runtime/native-dead-controller-result-lease.ts @@ -35,6 +35,7 @@ export async function releaseDeadControllerResultLeases(db: Db, input: { leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, controllerBootId: nativeRunFinalizations.controllerBootId, controllerPid: nativeRunFinalizations.controllerPid, + controllerHostHash: nativeRunFinalizations.controllerHostHash, controllerProcessStartedAt: nativeRunFinalizations.controllerProcessStartedAt, }).from(nativeRunFinalizations) .innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, nativeRunFinalizations.runId), diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index 52d8d15c9b7..eaa8708e510 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -1,3 +1,4 @@ +import { currentProcessSpaceHash } from "../controller-process-identity.js"; import { describe, expect, it, vi } from "vitest"; import { @@ -269,6 +270,25 @@ describe("native controller takeover fencing", () => { }); }); + it.each([ + ["another host", "another-host-pid-space"], + ["an unreadable PID space", "unknown"], + ])("never treats a dead or recycled local PID as proof for a controller on %s", async (_name, controllerHostHash) => { + for (const probe of [{ isProcessAlive: () => false }, { isProcessAlive: () => true, + readProcessStartedAt: async () => new Date("2026-09-04T11:30:00.000Z") }]) { + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash }, now, ...probe })) + .resolves.toEqual({ allowed: false, reason: "controller_host_unprovable" }); + } + }); + + it("proves a controller gone only in its own recorded PID space", async () => { + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: currentProcessSpaceHash() }, + now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); + // A row saved before the PID space was recorded keeps the previous local proof for one release. + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: null }, + now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); + }); + it.each([true, false])("requires a stopped idle controller even after its run lease was released: alive=%s", async alive => { await expect(evaluateNativeControllerTakeover({ owner: owner({ leaseOwner: null, leaseExpiresAt: null }), now, requireStoppedController: true, diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 54c4087d299..98d8444ee24 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -35,7 +35,7 @@ import { } from "@paperclipai/db"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { readProcessStartedAt } from "../hot-restart.js"; -import { currentControllerBootId, currentProcessStartedAtMs, processStartsDefinitelyDiffer } from "../controller-process-identity.js"; +import { currentControllerBootId, currentProcessSpaceHash, currentProcessStartedAtMs, processStartsDefinitelyDiffer, sameProcessSpace } from "../controller-process-identity.js"; import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; import { isUniqueViolation } from "../../db-errors.js"; @@ -54,6 +54,9 @@ export type NativeControllerIdentity = { bootId: string; pid: number; processStartedAt: Date; + /** PID space of `pid`; "unknown" when this host's space cannot be read, + * which no later probe accepts. Optional for identities built in tests. */ + hostHash?: string; }; export type NativeRestartKind = "hot" | "hard" | "graceful"; @@ -334,6 +337,7 @@ export async function currentNativeControllerIdentity(): Promise; + > & { + /** Absent for owners without a stored PID space (receipts, rows saved + * before the column existed): those keep the previous local proof. */ + controllerHostHash?: string | null; + }; now: Date; coordinatedPreviousController?: { pid: number; @@ -396,6 +404,12 @@ export async function evaluateNativeControllerTakeover(input: { : "live_legacy_lease_without_controller_identity", }; } + // A PID can only be judged in the PID space that recorded it. A controller + // on another host or namespace (or an unreadable local space) is never + // proven gone by a local probe; its lease or fence decides instead. + if (owner.controllerHostHash && !sameProcessSpace(owner.controllerHostHash)) { + return { allowed: false, reason: "controller_host_unprovable" }; + } if (!isAlive(priorPid)) { return { allowed: true, @@ -1522,6 +1536,7 @@ export async function claimNativeRestartRecoveries(input: { leaseExpiresAt: sql`clock_timestamp() + ${durableAuthority ? REMOTE_CONTROLLER_LEASE_MS : 20 * 60_000} * interval '1 millisecond'`, controllerBootId: controller.bootId, controllerPid: controller.pid, + controllerHostHash: controller.hostHash ?? null, controllerProcessStartedAt: controller.processStartedAt, controllerGeneration: generation, recoveryState: diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 4ee6b2a0305..4e18dae27f2 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -9831,6 +9831,7 @@ async function executePaperclipNativeSessionWithinScope( leaseExpiresAt: sql`clock_timestamp() + ${executionLeaseTtlMs} * interval '1 millisecond'`, controllerBootId: controller.bootId, controllerPid: controller.pid, + controllerHostHash: controller.hostHash ?? null, controllerProcessStartedAt: controller.processStartedAt, controllerGeneration: nextControllerGeneration, failureCode: null, diff --git a/server/src/services/workspace-operations.ts b/server/src/services/workspace-operations.ts index acf39177275..ab2aa012b7f 100644 --- a/server/src/services/workspace-operations.ts +++ b/server/src/services/workspace-operations.ts @@ -11,6 +11,7 @@ import { asc, desc, eq, gte, inArray, isNull, lt, or, and } from "drizzle-orm"; import { conflict, notFound } from "../errors.js"; import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js"; import { instanceSettingsService } from "./instance-settings.js"; +import { currentProcessSpaceHash, processExists, sameProcessSpace } from "./controller-process-identity.js"; import { getWorkspaceOperationLogStore } from "./workspace-operation-log-store.js"; type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect; @@ -71,6 +72,8 @@ type RuntimeControlOwnerStamp = { pid: number; action: string; heartbeatAt: string; + /** PID space of `pid`. Absent on stamps written before it was recorded. */ + hostHash?: string; }; function readRuntimeControlAction(metadata: unknown): string | null { @@ -91,6 +94,7 @@ function readRuntimeControlOwner(metadata: unknown): RuntimeControlOwnerStamp | pid: typeof record.pid === "number" ? record.pid : 0, action: typeof record.action === "string" ? record.action : "start", heartbeatAt: typeof record.heartbeatAt === "string" ? record.heartbeatAt : new Date(0).toISOString(), + ...(typeof record.hostHash === "string" ? { hostHash: record.hostHash } : {}), }; } @@ -118,20 +122,17 @@ export function isNativeWorkspaceFinalizationOperationActive(scope: NativeFinali } /** - * Whether the process that stamped an operation is still alive on this host. A dead owner can - * never heartbeat again, so its operation is recoverable immediately instead of after the - * staleness window — which is what makes recovery after a crash or restart prompt. Signal 0 - * only probes; it never touches the process. + * Whether the process that stamped an operation has provably exited. A dead owner can never + * heartbeat again, so its operation is recoverable immediately instead of after the staleness + * window — which is what makes recovery after a crash or restart prompt. The PID is judged only + * in the PID space that recorded it: an owner on another host or namespace, an unreadable + * space, or a stamp without a recorded space is never proven gone here and waits out the + * staleness window instead. Signal 0 only probes; EPERM means the process exists. */ -function ownerProcessIsAlive(pid: number): boolean { - if (!Number.isInteger(pid) || pid <= 0) return false; - try { - process.kill(pid, 0); - return true; - } catch (error) { - // EPERM means the pid exists but belongs to another user: treat it as alive. - return (error as NodeJS.ErrnoException)?.code === "EPERM"; - } +function ownerProcessProvenGone(owner: RuntimeControlOwnerStamp): boolean { + if (!sameProcessSpace(owner.hostHash)) return false; + if (!Number.isInteger(owner.pid) || owner.pid <= 0) return false; + return !processExists(owner.pid); } /** @@ -336,7 +337,7 @@ export function workspaceOperationService(db: Db) { const ownedHereAndLive = ownedByThisProcess && liveRuntimeControlOperationIds.has(candidate.id); // A previous server process that no longer exists cannot heartbeat again, so its // operations are recoverable at once instead of after the staleness window. - const ownerProcessGone = owner !== null && !ownedByThisProcess && !ownerProcessIsAlive(owner.pid); + const ownerProcessGone = owner !== null && !ownedByThisProcess && ownerProcessProvenGone(owner); if (ownedHereAndLive) { // This process is driving it right now; its own time budget is what terminalizes it. @@ -543,6 +544,7 @@ export function workspaceOperationService(db: Db) { runtimeControlOwner: { ownerId: RUNTIME_CONTROL_OWNER_ID, pid: process.pid, + hostHash: currentProcessSpaceHash(), action: runtimeControlAction, heartbeatAt: startedAt.toISOString(), } satisfies RuntimeControlOwnerStamp, @@ -594,6 +596,7 @@ export function workspaceOperationService(db: Db) { runtimeControlOwner: { ownerId: RUNTIME_CONTROL_OWNER_ID, pid: process.pid, + hostHash: currentProcessSpaceHash(), action: runtimeControlAction, heartbeatAt: heartbeatAt.toISOString(), } satisfies RuntimeControlOwnerStamp,