fix(runtime): authenticate provider process groups

Give each guarded Codex launch a random ownership token inherited by every descendant. Process-group recovery now enumerates live Linux members and signals only a group whose complete membership carries the persisted token, preventing cleanup after numeric group-ID reuse.

Co-authored-by: Codex <codex@openai.com>

packages/db/src/migrations/0230_lowly_stranger.sql

@@ -0,0 +1 @@
+ALTER TABLE "heartbeat_runs" ADD COLUMN "provider_process_owner_token" text;

packages/db/src/migrations/meta/_journal.json

@@ -1597,6 +1597,13 @@
       "when": 1787775574417,
       "tag": "0229_wise_amphibian",
       "breakpoints": true
+    },
+    {
+      "idx": 230,
+      "version": "7",
+      "when": 1787776668722,
+      "tag": "0230_lowly_stranger",
+      "breakpoints": true
     }
   ]
 }

packages/db/src/schema/heartbeat_runs.ts

@@ -66,6 +66,7 @@ export const heartbeatRuns = pgTable(
     providerProcessPid: integer("provider_process_pid"),
     providerProcessGroupId: integer("provider_process_group_id"),
     providerProcessStartedAt: timestamp("provider_process_started_at", { withTimezone: true }),
+    providerProcessOwnerToken: text("provider_process_owner_token"),
     lastOutputAt: timestamp("last_output_at", { withTimezone: true }),
     lastOutputSeq: integer("last_output_seq").notNull().default(0),
     lastOutputStream: text("last_output_stream"),

server/src/__tests__/heartbeat-process-recovery.test.ts

@@ -492,6 +492,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
     providerProcessPid?: number | null;
     providerProcessGroupId?: number | null;
     providerProcessStartedAt?: Date | null;
+    providerProcessOwnerToken?: string | null;
     processLossRetryCount?: number;
     includeIssue?: boolean;
     runErrorCode?: string | null;
@@ -556,6 +557,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
       providerProcessGroupId: input?.providerProcessGroupId ?? null,
       providerProcessStartedAt: input?.providerProcessStartedAt
         ?? (input?.providerProcessPid ? now : null),
+      providerProcessOwnerToken: input?.providerProcessOwnerToken ?? null,
       processLossRetryCount: input?.processLossRetryCount ?? 0,
       errorCode: input?.runErrorCode ?? null,
       error: input?.runError ?? null,
@@ -1268,6 +1270,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
       providerProcessPid: null,
       providerProcessGroupId: null,
       providerProcessStartedAt: null,
+      providerProcessOwnerToken: null,
     });
   });
 
@@ -1294,14 +1297,20 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
       providerProcessPid: null,
       providerProcessGroupId: null,
       providerProcessStartedAt: null,
+      providerProcessOwnerToken: null,
     });
   });
 
   it("terminates provider-group descendants after the guardian exits", async () => {
+    const ownerToken = randomUUID();
     const guardian = spawn(
       "/bin/sh",
       ["-c", "sleep 3600 & echo $!; sleep 0.2"],
-      { detached: true, stdio: ["ignore", "pipe", "ignore"] },
+      {
+        detached: true,
+        env: { ...process.env, PAPERCLIP_PROVIDER_OWNER_TOKEN: ownerToken },
+        stdio: ["ignore", "pipe", "ignore"],
+      },
     );
     childProcesses.add(guardian);
     expect(guardian.pid).toBeTypeOf("number");
@@ -1324,6 +1333,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
       providerProcessPid: guardian.pid ?? null,
       providerProcessGroupId: guardian.pid ?? null,
       providerProcessStartedAt: new Date(guardianStartedAt ?? 0),
+      providerProcessOwnerToken: ownerToken,
       includeIssue: false,
     });
     const heartbeat = heartbeatService(db);
@@ -1338,6 +1348,56 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
       providerProcessPid: null,
       providerProcessGroupId: null,
       providerProcessStartedAt: null,
+      providerProcessOwnerToken: null,
+    });
+  });
+
+  it("does not signal a reused provider group with a different owner token", async () => {
+    const replacementToken = randomUUID();
+    const replacementLeader = spawn(
+      "/bin/sh",
+      ["-c", "sleep 3600 & echo $!; sleep 0.2"],
+      {
+        detached: true,
+        env: { ...process.env, PAPERCLIP_PROVIDER_OWNER_TOKEN: replacementToken },
+        stdio: ["ignore", "pipe", "ignore"],
+      },
+    );
+    childProcesses.add(replacementLeader);
+    expect(replacementLeader.pid).toBeTypeOf("number");
+    const descendantPid = await new Promise<number>((resolvePid, rejectPid) => {
+      replacementLeader.stdout?.once("data", (chunk: Buffer) => {
+        const parsedPid = Number.parseInt(chunk.toString("utf8").trim(), 10);
+        if (Number.isInteger(parsedPid) && parsedPid > 0) resolvePid(parsedPid);
+        else rejectPid(new Error("Replacement descendant pid was not emitted"));
+      });
+      replacementLeader.once("error", rejectPid);
+    });
+    cleanupPids.add(descendantPid);
+    await new Promise<void>((resolveExit) => replacementLeader.once("exit", () => resolveExit()));
+    expect(isPidAlive(descendantPid)).toBe(true);
+
+    const { runId } = await seedRunFixture({
+      processPid: 999_999_999,
+      providerProcessPid: replacementLeader.pid ?? null,
+      providerProcessGroupId: replacementLeader.pid ?? null,
+      providerProcessStartedAt: new Date("2020-01-01T00:00:00.000Z"),
+      providerProcessOwnerToken: randomUUID(),
+      includeIssue: false,
+    });
+    const heartbeat = heartbeatService(db);
+
+    const result = await heartbeat.reapOrphanedRuns();
+
+    expect(result.reaped).toBe(1);
+    expect(isPidAlive(descendantPid)).toBe(true);
+    const run = await heartbeat.getRun(runId);
+    expect(run).toMatchObject({
+      status: "failed",
+      providerProcessPid: null,
+      providerProcessGroupId: null,
+      providerProcessStartedAt: null,
+      providerProcessOwnerToken: null,
     });
   });
 

server/src/services/heartbeat.ts

@@ -6394,6 +6394,33 @@ function isProcessAlive(pid: number | null | undefined) {
   }
 }
 
+async function isOwnedProviderProcessGroup(
+  processGroupId: number,
+  ownerToken: string,
+): Promise<boolean> {
+  if (process.platform !== "linux" || !ownerToken) return false;
+  const entries = await fs.readdir("/proc", { withFileTypes: true }).catch(() => []);
+  const memberPids: number[] = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue;
+    const pid = Number.parseInt(entry.name, 10);
+    const stat = await fs.readFile(`/proc/${pid}/stat`, "utf8").catch(() => null);
+    if (!stat) continue;
+    const commandEnd = stat.lastIndexOf(")");
+    if (commandEnd < 0) continue;
+    const fields = stat.slice(commandEnd + 1).trim().split(/\s+/);
+    const observedGroupId = Number.parseInt(fields[2] ?? "", 10);
+    if (observedGroupId === processGroupId) memberPids.push(pid);
+  }
+  if (memberPids.length === 0) return false;
+  const expectedEntry = `PAPERCLIP_PROVIDER_OWNER_TOKEN=${ownerToken}`;
+  const ownership = await Promise.all(memberPids.map(async (pid) => {
+    const environ = await fs.readFile(`/proc/${pid}/environ`).catch(() => null);
+    return environ?.toString("utf8").split("\0").includes(expectedEntry) ?? false;
+  }));
+  return ownership.every(Boolean);
+}
+
 async function terminateHeartbeatRunProcess(input: {
   pid: number | null | undefined;
   processGroupId: number | null | undefined;
@@ -9909,7 +9936,12 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
 
   async function persistRunProviderProcessMetadata(
     runId: string,
-    meta: { pid: number; processGroupId: number | null; startedAt: string },
+    meta: {
+      pid: number;
+      processGroupId: number | null;
+      startedAt: string;
+      ownerToken: string;
+    },
   ) {
     const startedAt = new Date(meta.startedAt);
     return db
@@ -9918,6 +9950,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
         providerProcessPid: meta.pid,
         providerProcessGroupId: meta.processGroupId,
         providerProcessStartedAt: Number.isNaN(startedAt.getTime()) ? new Date() : startedAt,
+        providerProcessOwnerToken: meta.ownerToken,
         updatedAt: new Date(),
       })
       .where(eq(heartbeatRuns.id, runId))
@@ -9932,6 +9965,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
         providerProcessPid: null,
         providerProcessGroupId: null,
         providerProcessStartedAt: null,
+        providerProcessOwnerToken: null,
         updatedAt: new Date(),
       })
       .where(eq(heartbeatRuns.id, runId))
@@ -9942,7 +9976,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
   async function terminateRunProviderProcess(
     run: Pick<
       typeof heartbeatRuns.$inferSelect,
-      "id" | "providerProcessPid" | "providerProcessGroupId" | "providerProcessStartedAt"
+      | "id"
+      | "providerProcessPid"
+      | "providerProcessGroupId"
+      | "providerProcessStartedAt"
+      | "providerProcessOwnerToken"
     >,
   ): Promise<boolean> {
     const pid = run.providerProcessPid;
@@ -9954,12 +9992,15 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
       const observedStartedAt = await readProcessStartedAt(pid).catch(() => null);
       ownsLiveProcess = observedStartedAt !== null
         && Date.parse(observedStartedAt) === run.providerProcessStartedAt.getTime();
-    } else if (processGroupAlive && pid && processGroupId === pid) {
-      // The persisted guardian is the process-group leader. POSIX keeps that
-      // group id allocated while any original descendant remains. If the id
-      // has been reused by a new live leader, the branch above validates its
-      // start identity instead.
-      ownsLiveProcess = true;
+    } else if (
+      processGroupAlive
+      && processGroupId
+      && run.providerProcessOwnerToken
+    ) {
+      ownsLiveProcess = await isOwnedProviderProcessGroup(
+        processGroupId,
+        run.providerProcessOwnerToken,
+      );
     }
     if (ownsLiveProcess) {
       await terminateHeartbeatRunProcess({ pid, processGroupId });

server/src/services/native-runtime/native-codex-runner.ts

@@ -1,5 +1,5 @@
 import { spawn, type ChildProcess } from "node:child_process";
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { accessSync, chmodSync, constants, existsSync, mkdirSync, readFileSync } from "node:fs";
 import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
@@ -144,6 +144,7 @@ async function startGuardedProvider(input: {
     pid: number;
     processGroupId: number;
     startedAt: string;
+    ownerToken: string;
   }) => Promise<void>;
 }): Promise<{
   guardian: ChildProcess;
@@ -153,13 +154,18 @@ async function startGuardedProvider(input: {
   if (process.platform === "win32") {
     throw new Error("codex_app_server_guardian_unsupported_platform");
   }
+  const ownerToken = randomUUID();
   const guardian = spawn(
     "/bin/sh",
     ["-c", PROVIDER_GUARDIAN_SCRIPT, "paperclip-codex-provider", input.command, ...input.args],
     {
       cwd: input.cwd,
       detached: true,
-      env: { ...process.env, ...input.environment },
+      env: {
+        ...process.env,
+        ...input.environment,
+        PAPERCLIP_PROVIDER_OWNER_TOKEN: ownerToken,
+      },
       stdio: ["pipe", "pipe", "pipe"],
     },
   );
@@ -172,6 +178,7 @@ async function startGuardedProvider(input: {
       pid: guardian.pid,
       processGroupId: guardian.pid,
       startedAt: ownerStartedAt,
+      ownerToken,
     });
 
     let buffer = "";
@@ -290,6 +297,7 @@ export async function executeNativeCodexRunner(input: {
     pid: number;
     processGroupId: number;
     startedAt: string;
+    ownerToken: string;
   }) => Promise<void>;
   /** Clears durable provider ownership after the shared server has stopped. */
   onProviderExit?: () => Promise<void>;