From bfed0ca90d4a16a67b909476fbe2669d600e98a3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 26 Aug 2026 20:26:45 +0000 Subject: [PATCH] fix(runtime): validate provider process identity Persist the operating-system start timestamp for the detached Codex provider and require an exact identity match before signaling its PID or process group. Clear stale ownership metadata without killing a replacement process, with focused PID-reuse coverage. Co-authored-by: Codex --- .../heartbeat-process-recovery.test.ts | 32 ++++++++++++++++++- server/src/services/heartbeat.ts | 13 ++++++-- .../native-runtime/native-codex-runner.ts | 7 +++- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index a959ca76df4..d3c5767c03b 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -108,6 +108,7 @@ import { } from "../services/heartbeat.ts"; import { readHotRestartIntent, + readProcessStartedAt, resolveLegacyHotRestartIntentPath, resolveHotRestartReportPath, writeHotRestartIntent, @@ -490,6 +491,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { processGroupId?: number | null; providerProcessPid?: number | null; providerProcessGroupId?: number | null; + providerProcessStartedAt?: Date | null; processLossRetryCount?: number; includeIssue?: boolean; runErrorCode?: string | null; @@ -552,7 +554,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { processGroupId: input?.processGroupId ?? null, providerProcessPid: input?.providerProcessPid ?? null, providerProcessGroupId: input?.providerProcessGroupId ?? null, - providerProcessStartedAt: input?.providerProcessPid ? now : null, + providerProcessStartedAt: input?.providerProcessStartedAt + ?? (input?.providerProcessPid ? now : null), processLossRetryCount: input?.processLossRetryCount ?? 0, errorCode: input?.runErrorCode ?? null, error: input?.runError ?? null, @@ -1250,6 +1253,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const { runId } = await seedRunFixture({ processPid: 999_999_999, providerProcessPid: provider.pid ?? null, + providerProcessStartedAt: new Date(await readProcessStartedAt(provider.pid ?? 0) ?? 0), includeIssue: false, }); const heartbeat = heartbeatService(db); @@ -1267,6 +1271,32 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }); }); + it("does not signal a reused provider pid with a different start time", async () => { + const replacement = spawnAliveProcess(); + childProcesses.add(replacement); + expect(replacement.pid).toBeTypeOf("number"); + + const { runId } = await seedRunFixture({ + processPid: 999_999_999, + providerProcessPid: replacement.pid ?? null, + providerProcessStartedAt: new Date("2020-01-01T00:00:00.000Z"), + includeIssue: false, + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reapOrphanedRuns(); + + expect(result.reaped).toBe(1); + expect(isPidAlive(replacement.pid)).toBe(true); + const run = await heartbeat.getRun(runId); + expect(run).toMatchObject({ + status: "failed", + providerProcessPid: null, + providerProcessGroupId: null, + providerProcessStartedAt: null, + }); + }); + it("skips generic timer wakes without invoking an adapter when no assigned work is actionable", async () => { const { companyId, agentId } = await seedIdleTimerAgentFixture(); const heartbeat = heartbeatService(db); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 57f146e6365..fa90013938b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -313,6 +313,7 @@ import { import { findMissingHotRestartSnapshotRunIds, readHotRestartIntent, + readProcessStartedAt, removeHotRestartIntent, shouldHonorHotRestartIntentForProcess, writeHotRestartReport, @@ -9941,19 +9942,25 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function terminateRunProviderProcess( run: Pick< typeof heartbeatRuns.$inferSelect, - "id" | "providerProcessPid" | "providerProcessGroupId" + "id" | "providerProcessPid" | "providerProcessGroupId" | "providerProcessStartedAt" >, ): Promise { const pid = run.providerProcessPid; const processGroupId = run.providerProcessGroupId; const alive = isProcessAlive(pid) || isProcessGroupAlive(processGroupId); - if (alive) { + let ownsLiveProcess = false; + if (alive && pid && run.providerProcessStartedAt) { + const observedStartedAt = await readProcessStartedAt(pid).catch(() => null); + ownsLiveProcess = observedStartedAt !== null + && Date.parse(observedStartedAt) === run.providerProcessStartedAt.getTime(); + } + if (ownsLiveProcess) { await terminateHeartbeatRunProcess({ pid, processGroupId }); } if (pid || processGroupId) { await clearRunProviderProcessMetadata(run.id); } - return alive; + return ownsLiveProcess; } async function clearDetachedRunWarning(runId: string) { diff --git a/server/src/services/native-runtime/native-codex-runner.ts b/server/src/services/native-runtime/native-codex-runner.ts index efe91b9165f..6c54af54661 100644 --- a/server/src/services/native-runtime/native-codex-runner.ts +++ b/server/src/services/native-runtime/native-codex-runner.ts @@ -10,6 +10,7 @@ import type { AdapterExecutionResult } from "@paperclipai/adapter-utils"; import type { Db } from "@paperclipai/db"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; +import { readProcessStartedAt } from "../hot-restart.js"; import { runnerPrpCoordinator } from "./runner-prp-coordinator.js"; const moduleDirectory = dirname(fileURLToPath(import.meta.url)); @@ -286,10 +287,14 @@ export async function executeNativeCodexRunner(input: { stderr = `${stderr}${chunk}`.slice(-16_384); }); if (!sharedCodexServer.pid) throw new Error("codex_app_server_process_not_started"); + const providerStartedAt = await readProcessStartedAt(sharedCodexServer.pid); + if (!providerStartedAt) { + throw new Error("codex_app_server_process_identity_unavailable"); + } await input.onProviderSpawn?.({ pid: sharedCodexServer.pid, processGroupId: process.platform === "win32" ? null : sharedCodexServer.pid, - startedAt: new Date().toISOString(), + startedAt: providerStartedAt, }); const startupDeadline = Date.now() + 10_000; while (