diff --git a/server/src/services/controller-process-identity.ts b/server/src/services/controller-process-identity.ts index 2bf79735f6f..1a537a1eb0b 100644 --- a/server/src/services/controller-process-identity.ts +++ b/server/src/services/controller-process-identity.ts @@ -10,6 +10,15 @@ import { readProcessStartedAt } from "./hot-restart.js"; */ export type ControllerProcess = { bootId: string; pid: number; hostHash: string; processStartedAt: string | null }; +/** + * The one boot id of this server process. Native coordinator claims + * (`native_run_finalizations.controller_boot_id`), legacy controller leases + * (`heartbeat_runs.controller_boot_id`), sandbox creation intents and teardown + * claims all stamp this value, so "owned by this process" means the same thing + * in every table. Records written by an older release carry that release's + * random per-module ids; those never equal this value, exactly as a record + * from any earlier boot never did. + */ const bootId = randomUUID(); const UNKNOWN_PROCESS_SPACE = "unknown"; @@ -38,7 +47,6 @@ function processSpaceHash(): string { } const hostHash = processSpaceHash(); -let processStartedAt: Promise | undefined; async function observedProcessStartedAt(pid: number): Promise { try { @@ -49,14 +57,38 @@ async function observedProcessStartedAt(pid: number): Promise { } } +// This process's own start time never changes, so it is read from the OS once. +// On macOS the read runs `ps -o lstart`; uncached, it ran on every native +// claim, preparation check and lease renewal. A failed read is not cached, so +// a transient failure is retried by the next caller. +let ownStartedAt: Promise | undefined; + +/** OS-observed start time of this server process (epoch ms), read at most once + * successfully per process. Null when the OS cannot report it. */ +export function currentProcessStartedAtMs(): Promise { + if (!ownStartedAt) { + const read = observedProcessStartedAt(process.pid); + ownStartedAt = read; + void read.then(value => { + if (value === null && ownStartedAt === read) ownStartedAt = undefined; + }); + } + return ownStartedAt; +} + +/** Test seam: forget the cached start time so a test can stub the OS read. */ +export function resetCurrentProcessStartedAtForTests(): void { + ownStartedAt = undefined; +} + export function currentControllerBootId(): string { return bootId; } export async function currentControllerProcess(): Promise { - processStartedAt ??= observedProcessStartedAt(process.pid) - .then(value => value === null ? null : new Date(value).toISOString()); - return { bootId, pid: process.pid, hostHash, processStartedAt: await processStartedAt }; + const startedAt = await currentProcessStartedAtMs(); + return { bootId, pid: process.pid, hostHash, + processStartedAt: startedAt === null ? null : new Date(startedAt).toISOString() }; } export function readControllerProcess(value: unknown): ControllerProcess | null { diff --git a/server/src/services/legacy-controller-lease.ts b/server/src/services/legacy-controller-lease.ts index a6a281db059..ae7656429b9 100644 --- a/server/src/services/legacy-controller-lease.ts +++ b/server/src/services/legacy-controller-lease.ts @@ -1,11 +1,13 @@ import { notifyActiveExecutionRecovery, notifyLiveRunRecovery } from "./live-run-recovery-notifications.js"; -import { randomUUID } from "node:crypto"; import { and, eq, gt, lte, sql } from "drizzle-orm"; import { heartbeatRuns, livenessDb, type Db } from "@paperclipai/db"; import type { LegacyControllerLeaseLossCause, LegacyControllerLeaseLossTiming } from "./run-cancellation.js"; +import { randomUUID } from "node:crypto"; +import { currentControllerBootId } from "./controller-process-identity.js"; -// A boot UUID has meaning across containers; a numeric PID does not. -export const legacyControllerBootId = randomUUID(); +// A boot UUID has meaning across containers; a numeric PID does not. Legacy +// leases stamp the single process boot id shared with native claims. +export const legacyControllerBootId = currentControllerBootId(); export const LEGACY_CONTROLLER_LEASE_MS = 60_000; export const LEGACY_CONTROLLER_RENEW_MS = 10_000; diff --git a/server/src/services/native-runtime/native-controller-identity.test.ts b/server/src/services/native-runtime/native-controller-identity.test.ts index fd11591da9a..fbd899aa498 100644 --- a/server/src/services/native-runtime/native-controller-identity.test.ts +++ b/server/src/services/native-runtime/native-controller-identity.test.ts @@ -2,7 +2,13 @@ import { afterEach, expect, it, vi } from "vitest"; import * as childProcess from "node:child_process"; import * as hotRestart from "../hot-restart.js"; import { getServerInfoSnapshot, resetServerInfoCacheForTests } from "../../server-info.js"; -import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; +import { currentNativeControllerIdentity, isCurrentNativeControllerProcess } from "./native-restart-recovery.js"; +import { + currentControllerBootId, + currentControllerProcess, + resetCurrentProcessStartedAtForTests, +} from "../controller-process-identity.js"; +import { legacyControllerBootId } from "../legacy-controller-lease.js"; vi.mock("node:child_process", async importOriginal => ({ ...await importOriginal(), @@ -14,6 +20,7 @@ vi.mock("node:child_process", async importOriginal => ({ afterEach(() => { vi.restoreAllMocks(); resetServerInfoCacheForTests(); + resetCurrentProcessStartedAtForTests(); }); it.each([true, false])("keeps controller identity independent of Git commands (OS start available=%s)", async available => { @@ -35,3 +42,39 @@ it.each([true, false])("keeps controller identity independent of Git commands (O expect(first.processStartedAt.toISOString()).toBe(available ? observed : fallback); expect(subprocess).not.toHaveBeenCalled(); }); + +it("stamps one boot id for native claims, legacy leases and creation intents", async () => { + vi.spyOn(hotRestart, "readProcessStartedAt").mockResolvedValue("2026-11-02T12:00:00.000Z"); + const native = await currentNativeControllerIdentity(); + const controller = await currentControllerProcess(); + + expect(native.bootId).toBe(currentControllerBootId()); + expect(legacyControllerBootId).toBe(currentControllerBootId()); + expect(controller.bootId).toBe(currentControllerBootId()); + expect(controller.processStartedAt).toBe(native.processStartedAt.toISOString()); + expect(isCurrentNativeControllerProcess({ controllerBootId: legacyControllerBootId, controllerPid: process.pid })).toBe(true); +}); + +it("reads this process's start time from the OS once across claims and renewals", async () => { + const read = vi.spyOn(hotRestart, "readProcessStartedAt").mockResolvedValue("2026-11-02T12:00:00.000Z"); + + await Promise.all(Array.from({ length: 5 }, () => currentNativeControllerIdentity())); + for (let i = 0; i < 5; i++) { + await currentNativeControllerIdentity(); + await currentControllerProcess(); + } + + expect(read).toHaveBeenCalledTimes(1); +}); + +it("retries a failed start-time read instead of caching the failure", async () => { + const read = vi.spyOn(hotRestart, "readProcessStartedAt") + .mockRejectedValueOnce(new Error("ps unavailable")) + .mockResolvedValue("2026-11-02T12:00:00.000Z"); + + expect((await currentControllerProcess()).processStartedAt).toBeNull(); + expect((await currentControllerProcess()).processStartedAt).toBe("2026-11-02T12:00:00.000Z"); + await currentNativeControllerIdentity(); + + expect(read).toHaveBeenCalledTimes(2); +}); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index f33e728ec2f..54c4087d299 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -35,7 +35,7 @@ import { } from "@paperclipai/db"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { readProcessStartedAt } from "../hot-restart.js"; -import { processStartsDefinitelyDiffer } from "../controller-process-identity.js"; +import { currentControllerBootId, currentProcessStartedAtMs, processStartsDefinitelyDiffer } from "../controller-process-identity.js"; import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; import { isUniqueViolation } from "../../db-errors.js"; @@ -282,7 +282,6 @@ export async function restartResumeKeepsAttempt(tx: Pick, input: { return work !== undefined; } -const controllerBootId = randomUUID(); function processIsAlive(pid: number | null | undefined): boolean { if (!pid || !Number.isInteger(pid) || pid <= 0) return false; @@ -324,14 +323,17 @@ async function observedProcessStart(pid: number): Promise { } } +/** The native controller stamp of this process. The boot id is the single + * process boot id (`controller-process-identity.ts`) and the start time is read + * from the OS once, so claims and renewals on the turn path spawn no `ps`. */ export async function currentNativeControllerIdentity(): Promise { - const observed = await observedProcessStart(process.pid); + const observed = await currentProcessStartedAtMs(); const fallback = new Date(getServerProcessStartedAt()); return { - bootId: controllerBootId, + bootId: currentControllerBootId(), pid: process.pid, processStartedAt: - observed ?? (Number.isNaN(fallback.getTime()) ? new Date() : fallback), + observed !== null ? new Date(observed) : (Number.isNaN(fallback.getTime()) ? new Date() : fallback), }; } @@ -339,7 +341,7 @@ export function isCurrentNativeControllerProcess(owner: { controllerBootId: string | null; controllerPid: number | null; }): boolean { - return owner.controllerBootId === controllerBootId && owner.controllerPid === process.pid; + return owner.controllerBootId === currentControllerBootId() && owner.controllerPid === process.pid; } function sameProcessStart(left: Date | null, right: Date | null): boolean {