diff --git a/server/src/services/controller-process-identity.test.ts b/server/src/services/controller-process-identity.test.ts index e029bf52589..2fd8b9a465f 100644 --- a/server/src/services/controller-process-identity.test.ts +++ b/server/src/services/controller-process-identity.test.ts @@ -4,7 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { processExists, processStartsDefinitelyDiffer } from "./controller-process-identity.js"; import { isPidAlive, isProcessGroupAlive } from "./local-service-supervisor.js"; import { readProcessStartedAt } from "./hot-restart.js"; -import { verifiedRecoveryProcessIsAlive } from "./native-runtime/native-session-executor.js"; +import { recoveryProcessMayBeAlive } from "./native-runtime/native-session-executor.js"; describe("shared process identity proof", () => { afterEach(() => vi.restoreAllMocks()); @@ -41,9 +41,14 @@ describe("shared process identity proof", () => { try { const observed = new Date((await readProcessStartedAt(live.pid!))!); const startedAt = new Date(observed.getTime() + 500).toISOString(); - await expect(verifiedRecoveryProcessIsAlive({ pid: live.pid!, startedAt })).resolves.toBe(true); - const recycled = new Date(observed.getTime() + 5_000).toISOString(); - await expect(verifiedRecoveryProcessIsAlive({ pid: live.pid!, startedAt: recycled })).resolves.toBe(false); + await expect(recoveryProcessMayBeAlive({ pid: live.pid!, startedAt })).resolves.toBe(true); + // The live PID started definitely after the recorded runner: reused, so gone. + const recycled = new Date(observed.getTime() - 5_000).toISOString(); + await expect(recoveryProcessMayBeAlive({ pid: live.pid!, startedAt: recycled })).resolves.toBe(false); + // A live PID that started before the recorded runner is an inconsistent + // stamp (shared prover), never exit evidence. + const inconsistent = new Date(observed.getTime() + 5_000).toISOString(); + await expect(recoveryProcessMayBeAlive({ pid: live.pid!, startedAt: inconsistent })).resolves.toBe(true); } finally { const exited = once(live, "exit"); live.kill("SIGKILL"); diff --git a/server/src/services/native-local-process-stop.test.ts b/server/src/services/native-local-process-stop.test.ts new file mode 100644 index 00000000000..1b649ba7c10 --- /dev/null +++ b/server/src/services/native-local-process-stop.test.ts @@ -0,0 +1,71 @@ +import { randomUUID } from "node:crypto"; +import { hostname } from "node:os"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { currentProcessSpaceHash, currentProcessStartedAtMs } from "./controller-process-identity.js"; +import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js"; +import { PROCESS_IDENTITY_RECORDED, recordedLocalExecutionStopped } from "./native-local-process-stop.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +// Phase 2: a bound local execution is judged by the shared prover +// (proveOwnerGone), in the PID space the row recorded. +(support.supported ? describe : describe.skip)("recordedLocalExecutionStopped", () => { + let database: Awaited>; + let db: ReturnType; + let companyId: string; + let agentId: string; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("native-local-stop-"); + db = createDb(database.connectionString, { maxConnections: 2 }); + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Local stop", issuePrefix: `L${companyId.slice(0, 7)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", role: "general", adapterType: "codex_local", + status: "idle" }); + }, 60_000); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + /** A failed run with its process identity bound by the launch event. */ + async function execution(input: { pid: number; startedAt: Date; processHostHash: string | null }) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", runtimeMode: "native", + processPid: input.pid, processGroupId: input.pid, processStartedAt: input.startedAt, + processHostHash: input.processHostHash }).returning(); + await appendHeartbeatRunEvent(db, { companyId, runId: run!.id, agentId, eventType: PROCESS_IDENTITY_RECORDED, + payload: { targetKind: "local", hostname: hostname(), processPid: input.pid, processGroupId: input.pid, + processStartedAt: input.startedAt.toISOString() } }); + return run!; + } + const deadPid = 2_000_000_001; + const minuteAgo = () => new Date(Date.now() - 60_000); + const ownStart = async () => new Date((await currentProcessStartedAtMs())!); + + it("is stopped when its dead PID was recorded in this PID space", async () => { + expect(await recordedLocalExecutionStopped(db, await execution({ pid: deadPid, startedAt: minuteAgo(), + processHostHash: currentProcessSpaceHash() }))).toBe(true); + }); + + it("is never stopped by a dead PID recorded in another PID space", async () => { + expect(await recordedLocalExecutionStopped(db, await execution({ pid: deadPid, startedAt: minuteAgo(), + processHostHash: "another-pid-space" }))).toBe(false); + }); + + it("keeps the host-bound proof for a row saved before its PID space", async () => { + expect(await recordedLocalExecutionStopped(db, await execution({ pid: deadPid, startedAt: minuteAgo(), + processHostHash: null }))).toBe(true); + }); + + it("is not stopped while the same process is alive", async () => { + expect(await recordedLocalExecutionStopped(db, await execution({ pid: process.pid, startedAt: await ownStart(), + processHostHash: currentProcessSpaceHash() }))).toBe(false); + }); + + it("is stopped when a later process reused its PID and leads the group", async () => { + // This test process started definitely after the recorded execution. + const recorded = new Date((await ownStart()).getTime() - 5_000); + // A group with this id is either empty or led by this later process, so + // it cannot hold the recorded execution's children. + expect(await recordedLocalExecutionStopped(db, await execution({ pid: process.pid, startedAt: recorded, + processHostHash: currentProcessSpaceHash() }))).toBe(true); + }); +}); diff --git a/server/src/services/native-local-process-stop.ts b/server/src/services/native-local-process-stop.ts index 5999ea523b6..47f4055bf11 100644 --- a/server/src/services/native-local-process-stop.ts +++ b/server/src/services/native-local-process-stop.ts @@ -7,6 +7,8 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio import { nativeSessionProviderEvidence } from "./native-runtime/native-session-resume.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import os from "node:os"; +import { currentProcessSpaceHash } from "./controller-process-identity.js"; +import { proveOwnerGone } from "./owner-proof.js"; export const PROCESS_START_REQUESTED = "native.process_start_requested"; export const PROCESS_IDENTITY_RECORDED = "native.process_identity_recorded"; @@ -26,7 +28,7 @@ function absent(pid: number): boolean { /** A fresh workspace is not process containment. Bind an OS observation to * the original host, process birth, and launch event before accepting it. */ export async function recordedLocalExecutionStopped(db: Db, run: Pick) { + "id" | "companyId" | "processPid" | "processGroupId" | "processStartedAt"> & { processHostHash?: string | null }) { const [identity] = await db.select({ eventType: heartbeatRunEvents.eventType, payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), inArray(heartbeatRunEvents.eventType, [PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED]), @@ -36,8 +38,15 @@ export async function recordedLocalExecutionStopped(db: Db, run: Pick { } }); + // Phase 2: run processes and controllers are judged by the shared prover. + it("never releases on a dead PID recorded in another PID space", async () => { + const { runId } = await nativeRun({ status: "failed", errorCode: "adapter_failed", + coordinator: { leaseOwner: `default:resume:${randomUUID()}`, leaseExpiresAt: new Date(0) } }); + await db.update(heartbeatRuns).set({ processPid: 2_000_000_001, processGroupId: null, + processStartedAt: new Date(Date.now() - 60_000), processHostHash: "another-pid-space" }) + .where(eq(heartbeatRuns.id, runId)); + await expect(releaseAbandonedNativeOwners(db)).resolves.toBe(0); + expect(await coordinator(runId)).toMatchObject({ phase: "observed" }); + await db.update(heartbeatRuns).set({ processHostHash: currentProcessSpaceHash() }).where(eq(heartbeatRuns.id, runId)); + await expect(releaseAbandonedNativeOwners(db)).resolves.toBe(1); + expect(await db.select().from(activityLog).where(and(eq(activityLog.runId, runId), + eq(activityLog.action, "legacy_host_unknown_dead_pid_proof")))).toEqual([]); + }); + + it("keeps the one-release fallback for a run process saved before its PID space, and records it", async () => { + const { runId } = await nativeRun({ status: "failed", errorCode: "adapter_failed", + coordinator: { leaseOwner: `default:resume:${randomUUID()}`, leaseExpiresAt: new Date(0) } }); + await db.update(heartbeatRuns).set({ processPid: 2_000_000_001, processGroupId: null, + processStartedAt: new Date(Date.now() - 60_000), processHostHash: null }).where(eq(heartbeatRuns.id, runId)); + await expect(releaseAbandonedNativeOwners(db)).resolves.toBe(1); + const proofs = await db.select().from(activityLog).where(and(eq(activityLog.runId, runId), + eq(activityLog.action, "legacy_host_unknown_dead_pid_proof"))); + expect(proofs.map(proof => proof.details)).toEqual([{ reason: "pid_dead", source: "native_failed_run_process" }]); + }); + // `ps -o lstart` on macOS reports whole seconds, while a recorded start can // carry milliseconds. A sub-second gap must not be read as a recycled PID. it.each(["controller", "run process"])("keeps the owner when the live %s start time differs by under a second", async (holder) => { diff --git a/server/src/services/native-runtime/native-failed-run-owner-release.ts b/server/src/services/native-runtime/native-failed-run-owner-release.ts index 3ea34cda26b..ef7ddff6456 100644 --- a/server/src/services/native-runtime/native-failed-run-owner-release.ts +++ b/server/src/services/native-runtime/native-failed-run-owner-release.ts @@ -2,15 +2,15 @@ import { and, asc, eq, gt, inArray, isNotNull, isNull, lte, or, sql } from "driz import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; import { adapterExecutionControls } from "../adapter-execution-control.js"; -import { processExists, processStartsDefinitelyDiffer } from "../controller-process-identity.js"; +import { proveOwnerGone, type LocalOwnerStamp } from "../owner-proof.js"; import { logActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; -import { readProcessStartedAt } from "../hot-restart.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { LEGACY_WORKSPACE_RECOVERY_KEY } from "../workspace-restore-recovery-state.js"; import { activeNativeExecutionRunIds } from "./native-active-runs.js"; -import { isCurrentNativeControllerProcess } from "./native-restart-recovery.js"; +import { isCurrentNativeControllerProcess, recordLegacyHostFallbackProof } from "./native-restart-recovery.js"; +import { nativeControllerOwnerStamp, runProcessOwnerStamp } from "./native-owner-stamps.js"; /** * A native run can fail outside the native session executor: environment lease @@ -85,51 +85,48 @@ export async function nextAbandonedNativeOwnerReleaseAt(db: Db): Promise { - if (!pid || processAbsent(pid)) return false; - if (!startedAt) return true; - const observed = await readProcessStartedAt(pid).catch(() => null); - return !processStartsDefinitelyDiffer(startedAt, observed); +/** True unless the shared prover proves the stamped owner gone. Only a `gone` + * verdict releases: a live, unprovable or foreign-space owner may be live. */ +async function ownerMayBeLive(stamp: LocalOwnerStamp, source: string, fallbacks: FallbackUse[]): Promise { + const proof = await proveOwnerGone(stamp); + if (proof.verdict !== "gone") return true; + if (proof.legacyHostFallback) fallbacks.push({ reason: proof.reason, source }); + return false; } /** * The run-level process evidence `agentExecutionsHaveStopped` requires. This - * process answers for itself from its in-memory execution registries. + * process answers for itself from its in-memory execution registries; a + * recorded adapter process (PID and group) is judged by the shared prover. */ -async function runProcessMayBeLive(run: Run, now: Date): Promise { +async function runProcessMayBeLive(run: Run, now: Date, fallbacks: FallbackUse[]): Promise { if (adapterExecutionControls.has(run.id) || activeNativeExecutionRunIds.has(run.id)) return true; if (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > now) return true; - // Same process identity rule as `agentExecutionsHaveStopped`: a recorded PID - // that now has another start time was recycled, and so was its own group. - let processAlive = Boolean(run.processPid && !processAbsent(run.processPid)); - let groupAlive = Boolean(run.processGroupId && !processAbsent(-run.processGroupId)); - if (processAlive && run.processStartedAt) { - const observed = await readProcessStartedAt(run.processPid!).catch(() => null); - if (processStartsDefinitelyDiffer(run.processStartedAt, observed)) { - processAlive = false; - if (run.processGroupId === run.processPid) groupAlive = false; - } - } - return processAlive || groupAlive; + if (!run.processPid && !run.processGroupId) return false; + return ownerMayBeLive(runProcessOwnerStamp(run, "native_failed_run_process", { legacyFallback: true }), + "native_failed_run_process", fallbacks); } /** An expired owner's recorded controller, unless this process is that controller. */ -async function controllerMayBeLive(coordinator: Coordinator): Promise { +async function controllerMayBeLive(coordinator: Coordinator, fallbacks: FallbackUse[]): Promise { if (isCurrentNativeControllerProcess(coordinator)) return false; // A previous boot of this process can share its PID, but cannot be its controller. if (coordinator.controllerPid === process.pid) return false; - return recordedProcessMayBeLive(coordinator.controllerPid, coordinator.controllerProcessStartedAt); + if (!coordinator.controllerPid) return false; + // Not this boot (checked above), so the prover must not treat its boot id as live. + return ownerMayBeLive({ ...nativeControllerOwnerStamp(coordinator, "native_failed_run_controller"), bootId: null }, + "native_failed_run_controller", fallbacks); } async function settleFailedRunOwner(db: Db, release: Release, now: Date): Promise { const publications: ActivityPublication[] = []; const settled = await db.transaction(async transaction => { const tx = transaction as unknown as Db; + const fallbacks: FallbackUse[] = []; await tx.execute(sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`); const [binding] = await tx.select({ companyId: nativeRunFinalizations.companyId, issueId: nativeRunFinalizations.issueId }) .from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, release.runId)); @@ -151,8 +148,9 @@ async function settleFailedRunOwner(db: Db, release: Release, now: Date): Promis || run.resultJson?.[LEGACY_WORKSPACE_RECOVERY_KEY] != null) return false; const [heldElsewhere] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) .where(and(eq(heartbeatRuns.id, run.id), or(unsettledRunEnvironmentLease, pendingNativeStop))).limit(1); - if (heldElsewhere || await runProcessMayBeLive(run, now) || await controllerMayBeLive(coordinator)) return false; - } else if (await runProcessMayBeLive(run, now)) { + if (heldElsewhere || await runProcessMayBeLive(run, now, fallbacks) + || await controllerMayBeLive(coordinator, fallbacks)) return false; + } else if (await runProcessMayBeLive(run, now, fallbacks)) { // The claim never reached the executor, but the run may still own a local process. return false; } @@ -194,6 +192,9 @@ async function settleFailedRunOwner(db: Db, release: Release, now: Date): Promis if (!updated) return false; await tx.update(heartbeatRuns).set({ nativePhase: "terminal_failure", nativePhaseUpdatedAt: now, updatedAt: now }) .where(eq(heartbeatRuns.id, run.id)); + for (const use of fallbacks) { + await recordLegacyHostFallbackProof(tx, run, { allowed: true, reason: use.reason, legacyHostFallback: true }, use.source); + } await logActivity(tx, { companyId: run.companyId, actorType: "system", actorId: ACTOR_ID, action: "native.failed_run_owner_released", entityType: "heartbeat_run", entityId: run.id, diff --git a/server/src/services/native-runtime/native-owner-stamps.ts b/server/src/services/native-runtime/native-owner-stamps.ts new file mode 100644 index 00000000000..8bd2bd9dfdc --- /dev/null +++ b/server/src/services/native-runtime/native-owner-stamps.ts @@ -0,0 +1,47 @@ +import type { heartbeatRuns, nativeRunFinalizations } from "@paperclipai/db"; +import type { LocalOwnerStamp } from "../owner-proof.js"; + +/** + * Owner stamps for the native records that `proveOwnerGone` judges (plan + * Phase 2). The ownership registry and the native recovery paths build their + * stamps here, so one record has one proof. + */ + +type ControllerRow = Pick & { + controllerHostHash?: string | null; + controllerGeneration?: number | null; +}; + +/** NRF records the controller's boot, PID, start time and, since #19868, its + * PID space (`controller_host_hash`). Rows saved before that keep the + * one-release local proof on single-host servers (`processSpaceUnrecorded`, + * recorded as `legacy_host_unknown_dead_pid_proof`, #19883). */ +export function nativeControllerOwnerStamp(row: ControllerRow, scope: string): LocalOwnerStamp { + return { + kind: "local", scope, generation: row.controllerGeneration ?? null, host: null, + processSpace: row.controllerHostHash ?? null, + processSpaceUnrecorded: row.controllerHostHash == null, + bootId: row.controllerBootId, pid: row.controllerPid, + processStartedAt: row.controllerProcessStartedAt?.toISOString() ?? null, processGroupId: null, + }; +} + +type RunProcessRow = Pick & { + processHostHash?: string | null; +}; + +/** heartbeat_runs records the adapter child's PID, group, start time and, + * since #19871, its PID space (`process_host_hash`, or "remote" for a sandbox + * PID, which a local probe never judges). With `legacyFallback`, a row saved + * before that column keeps the one-release local proof; without it, such a row + * is unprovable. */ +export function runProcessOwnerStamp(row: RunProcessRow, scope: string, + options: { legacyFallback?: boolean } = {}): LocalOwnerStamp { + return { + kind: "local", scope, generation: null, host: null, processSpace: row.processHostHash ?? null, + ...(options.legacyFallback ? { processSpaceUnrecorded: row.processHostHash == null } : {}), + bootId: null, pid: row.processPid, processStartedAt: row.processStartedAt?.toISOString() ?? null, + processGroupId: row.processGroupId, + }; +} diff --git a/server/src/services/native-runtime/native-restart-recovery.test.ts b/server/src/services/native-runtime/native-restart-recovery.test.ts index 2e3d55ea235..3f88219608a 100644 --- a/server/src/services/native-runtime/native-restart-recovery.test.ts +++ b/server/src/services/native-runtime/native-restart-recovery.test.ts @@ -270,6 +270,33 @@ describe("native controller takeover fencing", () => { }); }); + // Phase 2: the local branch is the shared prover (proveOwnerGone). + it("accepts a reused PID only when the live process started definitely later", async () => { + const live = { isProcessAlive: () => true }; + await expect(evaluateNativeControllerTakeover({ owner: owner(), now, ...live, + readProcessStartedAt: async () => new Date(recordedStart.getTime() + 5_000) })) + .resolves.toEqual({ allowed: true, reason: "controller_pid_recycled" }); + // An observed start before the recorded one is an inconsistent stamp, + // which the previous private proof accepted as recycled. + await expect(evaluateNativeControllerTakeover({ owner: owner(), now, ...live, + readProcessStartedAt: async () => new Date(recordedStart.getTime() - 5_000) })) + .resolves.toEqual({ allowed: false, reason: "controller_pid_start_inconsistent" }); + // Within the 1 s precision window it is the same live controller. + await expect(evaluateNativeControllerTakeover({ owner: owner(), now, ...live, + readProcessStartedAt: async () => new Date(recordedStart.getTime() + 500) })) + .resolves.toEqual({ allowed: false, reason: "controller_still_alive" }); + await expect(evaluateNativeControllerTakeover({ owner: owner(), now, ...live, + readProcessStartedAt: async () => null })) + .resolves.toEqual({ allowed: false, reason: "controller_still_alive" }); + }); + + it("judges a controller recorded in this PID space, and keeps the operator attestation", async () => { + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: currentProcessSpaceHash() }, + now, isProcessAlive: () => false })).resolves.toEqual({ allowed: true, reason: "controller_process_dead" }); + await expect(evaluateNativeControllerTakeover({ owner: { ...owner(), controllerHostHash: "operator_attested" }, + now, isProcessAlive: () => true })).resolves.toEqual({ allowed: true, reason: "operator_attested_controller_stopped" }); + }); + it.each([ ["another host", "another-host-pid-space"], ["an unreadable PID space", "unknown"], diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index b780294b4e2..a5c799ff64a 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -36,10 +36,10 @@ import { import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { PROVIDER_ATTEMPT_CREDIT_GENERATION_LIMIT, providerAttemptChargesBudget, providerAttemptWindowStart, readProviderAttemptOutcome } from "./provider-attempt-outcome.js"; import { readProcessStartedAt } from "../hot-restart.js"; -import { ephemeralServerEnabled } from "../ephemeral-server-mode.js"; +import { currentProcessProbe, proveOwnerGone, type ProcessProbe } from "../owner-proof.js"; import { persistActivity } from "../activity-log.js"; import { OPERATOR_ATTESTED_CONTROLLER_HOST, requestNativeControllerStopConfirmation } from "./native-controller-host-recovery.js"; -import { currentControllerBootId, currentProcessSpaceHash, currentProcessStartedAtMs, processStartsDefinitelyDiffer, sameProcessSpace } from "../controller-process-identity.js"; +import { currentControllerBootId, currentProcessSpaceHash, currentProcessStartedAtMs, processStartsDefinitelyDiffer } from "../controller-process-identity.js"; import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; import { isUniqueViolation } from "../../db-errors.js"; @@ -418,8 +418,6 @@ export async function evaluateNativeControllerTakeover(input: { ? { allowed: false, reason: "remote_controller_lease_live" } : { allowed: true, reason: "remote_controller_lease_released_or_expired" }; } - const isAlive = input.isProcessAlive ?? processIsAlive; - const readStartedAt = input.readProcessStartedAt ?? observedProcessStart; if ((!owner.leaseOwner || !owner.leaseExpiresAt) && !input.requireStoppedController) { return { allowed: true, reason: "unowned" }; } @@ -440,21 +438,32 @@ export async function evaluateNativeControllerTakeover(input: { if (recordsHost && owner.controllerHostHash === OPERATOR_ATTESTED_CONTROLLER_HOST) { return { allowed: true, reason: "operator_attested_controller_stopped" }; } - // A PID can only be judged in the PID space that recorded it. A controller - // on another host or namespace (or an unreadable local space) is never - // proven gone by a local probe; its lease or fence decides instead. - if (owner.controllerHostHash && !sameProcessSpace(owner.controllerHostHash)) { - return { allowed: false, reason: "controller_host_unprovable" }; - } - // A row saved before the host column existed. An ephemeral (multi-host) - // server can never assume it was written here; a board operator decides. - if (recordsHost && owner.controllerHostHash == null && ephemeralServerEnabled()) { - return { allowed: false, reason: "controller_host_unrecorded" }; - } - // Single-host servers keep the previous local proof for one release, and - // every use is recorded so the release that removes it can check for rows. - const fallback = recordsHost && owner.controllerHostHash == null ? { legacyHostFallback: true as const } : {}; - if (!isAlive(priorPid)) { + // The local branch is the shared prover (plan Phase 2): a PID is judged + // only in the PID space that recorded it, a reused PID must have started + // definitely later, and EPERM is alive. A coordinator row saved before the + // host column keeps the one-release local proof on single-host servers + // (`processSpaceUnrecorded`; each use is recorded by the caller as + // LEGACY_HOST_FALLBACK_EVENT), and an ephemeral server never judges it. + // Owners built from receipts that never stored a PID space are judged in + // this PID space: their callers checked the recorded host first. + const base = currentProcessProbe(); + const probe: ProcessProbe = { + ...base, + // No platform-deployment shortcut: this stamp records no worker. + deployment: null, + pidExists: input.isProcessAlive ?? base.pidExists, + readStartedAt: input.readProcessStartedAt ?? base.readStartedAt, + }; + const proof = await proveOwnerGone({ + kind: "local", scope: "native_controller", generation: null, host: null, + processSpace: recordsHost ? owner.controllerHostHash ?? null : probe.processSpace, + processSpaceUnrecorded: recordsHost && owner.controllerHostHash == null, + bootId: null, pid: priorPid, processStartedAt: owner.controllerProcessStartedAt?.toISOString() ?? null, + processGroupId: null, + }, probe); + const fallback = proof.legacyHostFallback ? { legacyHostFallback: true as const } : {}; + if (proof.verdict === "gone") { + if (proof.reason === "pid_reused") return { allowed: true, reason: "controller_pid_recycled", ...fallback }; return { allowed: true, reason: @@ -464,18 +473,14 @@ export async function evaluateNativeControllerTakeover(input: { ...fallback, }; } - - const observedStartedAt = await readStartedAt(priorPid); - if ( - owner.controllerProcessStartedAt && - observedStartedAt && - definitivelyDifferentProcessStart( - owner.controllerProcessStartedAt, - observedStartedAt, - ) - ) { - return { allowed: true, reason: "controller_pid_recycled", ...fallback }; + if (proof.reason === "process_space_unrecorded") return { allowed: false, reason: "controller_host_unrecorded" }; + if (proof.reason === "different_process_space" || proof.reason === "unknown_process_space" + || proof.reason === "different_host") { + return { allowed: false, reason: "controller_host_unprovable" }; } + // A live PID observed to start before the recorded controller is an + // inconsistent stamp, never proof of reuse. + if (proof.reason === "pid_start_inconsistent") return { allowed: false, reason: "controller_pid_start_inconsistent" }; const coordinated = input.coordinatedPreviousController; if ( diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index d7e3759d66f..4b5d5ecd027 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -302,8 +302,8 @@ import { noteNativeRunnerEvidence } from "./native-runner-evidence-signal.js"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; import { registerLiveRunnerGoalController } from "../runner-goal-control-broker.js"; import { applyRunnerGoalPrpEvent } from "../runner-goals.js"; -import { readProcessStartedAt } from "../hot-restart.js"; -import { processExists, processStartsDefinitelyDiffer } from "../controller-process-identity.js"; +import { currentProcessSpaceHash } from "../controller-process-identity.js"; +import { proveOwnerGone } from "../owner-proof.js"; import { NativeRunnerOwnershipUnverifiedError, isNativeRunnerOwnershipHeld, @@ -591,32 +591,33 @@ const nativeRuntimeRequestResolutions = new Map< NativeRuntimeRequestResolution >(); +/** The shared prover's verdict on a recovered local runner, recorded on this + * host by an earlier boot (no boot id, so this process is never assumed). */ +function proveRecoveryProcess(input: { pid: number; startedAt: string }) { + return proveOwnerGone({ kind: "local", scope: "native_recovery_runner", generation: null, host: null, + processSpace: currentProcessSpaceHash(), bootId: null, pid: input.pid, processStartedAt: input.startedAt, + processGroupId: null }); +} + /** - * True when the recorded runner still owns its PID. A start time within the - * shared precision window is the same process: `ps -o lstart` on macOS has - * whole-second precision, so an exact match would call a live runner gone. + * False only when the shared prover proves the recorded runner gone (its PID + * is dead or reused by a later process). The runner treats false as exit + * evidence, so an unreadable or inconsistent start time stays alive. */ -export async function verifiedRecoveryProcessIsAlive(input: { +export async function recoveryProcessMayBeAlive(input: { pid: number; startedAt: string; }): Promise { - if (!processExists(input.pid)) return false; - try { - const observed = await readProcessStartedAt(input.pid); - return ( - observed !== null && - !processStartsDefinitelyDiffer(input.startedAt, observed) - ); - } catch { - return false; - } + return (await proveRecoveryProcess(input)).verdict !== "gone"; } async function signalVerifiedRecoveryProcess( input: { pid: number; startedAt: string }, signal: NodeJS.Signals, ): Promise { - if (!(await verifiedRecoveryProcessIsAlive(input))) return false; + // Signal only the proven same process (start time within the shared + // precision window), never an ambiguous or reused PID. + if ((await proveRecoveryProcess(input)).verdict !== "alive") return false; try { return process.kill(input.pid, signal); } catch { @@ -5467,7 +5468,7 @@ const RETIREMENT_REASON_NAMES = { recoveryState: new Set(["blocked", "resuming_session", "awaiting_evidence", "awaiting_runner_reattach"]), controllerReason: new Set(["controller_still_alive", "coordinated_controller_still_alive", "remote_controller_lease_live", "expired_lease_without_controller_identity", "live_legacy_lease_without_controller_identity", - "controller_host_unprovable", "controller_host_unrecorded"]), + "controller_host_unprovable", "controller_host_unrecorded", "controller_pid_start_inconsistent"]), sandboxOutcome: new Set(["created", "resumed", "replacement"]), } as const; @@ -14664,7 +14665,7 @@ async function createRunnerdBackendWithinSessionClaim( target, binding: input.execution.binding, trace: input.trace, onLog: input.onLog, }).controlPlaneRegistration, adoptExistingRunner: remoteDot?.adoptExistingRunner ?? (recoveredProcess ? { ...recoveredProcess, - isAlive: () => verifiedRecoveryProcessIsAlive(recoveredProcess), + isAlive: () => recoveryProcessMayBeAlive(recoveredProcess), signal: (signal: NodeJS.Signals = "SIGTERM") => signalVerifiedRecoveryProcess(recoveredProcess, signal) } : undefined), }; const backendOptions = { @@ -16012,7 +16013,7 @@ async function createRunnerdBackendWithinSessionClaim( : target.kind === "local" && localRecoveryProcess ? { ...localRecoveryProcess, - isAlive: () => verifiedRecoveryProcessIsAlive(localRecoveryProcess), + isAlive: () => recoveryProcessMayBeAlive(localRecoveryProcess), signal: (signal: NodeJS.Signals) => signalVerifiedRecoveryProcess(localRecoveryProcess, signal), } diff --git a/server/src/services/quarantined-task-retry.test.ts b/server/src/services/quarantined-task-retry.test.ts index f08669a6e84..f87d720da71 100644 --- a/server/src/services/quarantined-task-retry.test.ts +++ b/server/src/services/quarantined-task-retry.test.ts @@ -17,6 +17,7 @@ import { getExecutionBlocker } from "./execution-blocker.js"; import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js"; import { instanceSettingsService } from "./instance-settings.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; +import { currentProcessStartedAtMs } from "./controller-process-identity.js"; import { nativeSha256 } from "./native-runtime/canonical.js"; import { verifyStoppedNativeSessionForContinuation } from "./native-runtime/native-session-executor.js"; import { requestTaskRetry } from "./task-retry.js"; @@ -185,7 +186,10 @@ describe("task Retry after inherited native cleanup quarantine", () => { })); } if (mode === "worker_alive") { - await db.update(heartbeatRuns).set({ processPid: process.pid, processGroupId: process.pid }) + // This test process, with its real start time: the shared prover then + // sees the same live worker, not a reused PID. + const startedAt = new Date((await currentProcessStartedAtMs())!); + await db.update(heartbeatRuns).set({ processPid: process.pid, processGroupId: process.pid, processStartedAt: startedAt }) .where(eq(heartbeatRuns.companyId, f.companyId)); const [owner] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.ownerId)); await appendHeartbeatRunEvent(db, { companyId: f.companyId, runId: f.ownerId, agentId: f.agentId, diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index 6bc510d34c4..848c0992fdf 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -33,6 +33,7 @@ import { } from "../native-runtime/native-accepted-result-lease-settlement.js"; import { settleClaimedWakeRequest } from "./wake-request-claimed-settlement.js"; import { settleAbandonedWorkspaceFinalizeAttempt } from "./workspace-finalize-attempt-settlement.js"; +import { nativeControllerOwnerStamp, runProcessOwnerStamp } from "../native-runtime/native-owner-stamps.js"; import { legacyControllerOwnerStamp, settleLegacyControllerLease } from "../legacy-controller-lease.js"; import { nativeLeaseHolderStamp, @@ -163,20 +164,8 @@ const runOwner = (runId: string | null | undefined): RecordOwner => runId ? { ki type NrfRow = typeof nativeRunFinalizations.$inferSelect; -/** NRF records the controller's boot, PID, start time and, since #19868, its - * PID space (`controller_host_hash`). Rows written before that have no space - * and stay unprovable for a foreign boot (inventory finding 2). */ -function nrfStamp(row: NrfRow, scope: string): LocalOwnerStamp { - return { - kind: "local", scope, generation: row.controllerGeneration ?? null, host: null, - processSpace: row.controllerHostHash ?? null, - // Rows saved before #19868 keep the one-release local proof on single-host - // servers (`legacy_host_unknown_dead_pid_proof`, #19883). - processSpaceUnrecorded: row.controllerHostHash == null, - bootId: row.controllerBootId, pid: row.controllerPid, - processStartedAt: row.controllerProcessStartedAt?.toISOString() ?? null, processGroupId: null, - }; -} +/** The NRF controller stamp, shared with native recovery (native-owner-stamps.ts). */ +const nrfStamp = (row: NrfRow, scope: string): LocalOwnerStamp => nativeControllerOwnerStamp(row, scope); function nrfKind(id: string, title: string, obligation: Obligation, where: SQL | undefined, owner: (row: NrfRow) => RecordOwner = row => ({ kind: "process", stamp: nrfStamp(row, id) }), @@ -256,16 +245,9 @@ function runKind(id: string, title: string, obligation: Obligation, where: SQL | }; } -/** heartbeat_runs records the adapter child's PID, start time and, since - * #19871, its PID space (`process_host_hash`). Older rows have no space and - * stay unprovable. */ -function runProcessStamp(row: RunRow, scope: string): LocalOwnerStamp { - return { - kind: "local", scope, generation: null, host: null, processSpace: row.processHostHash ?? null, bootId: null, - pid: row.processPid, processStartedAt: row.processStartedAt?.toISOString() ?? null, - processGroupId: row.processGroupId, - }; -} +/** The adapter child's stamp (native-owner-stamps.ts). Rows saved before + * `process_host_hash` stay unprovable here. */ +const runProcessStamp = (row: RunRow, scope: string): LocalOwnerStamp => runProcessOwnerStamp(row, scope); const legacyControllerLeaseWhere = and(eq(heartbeatRuns.runtimeMode, "legacy"), eq(heartbeatRuns.status, "running"), isNotNull(heartbeatRuns.controllerBootId), sql`${heartbeatRuns.controllerLeaseExpiresAt} > now()`);