From ff53eb84db16c24295782b78ae8275550adb5479 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 7 Nov 2026 13:04:55 -0600 Subject: [PATCH] feat(recovery): add OwnerStamp and proveOwnerGone (gone | alive | unprovable) (#19853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Restart recovery must decide when a dead process's in-progress record may be released > - About 31 sites decide "owner gone" today, with different rules for EPERM, PID reuse and other hosts (Phase 0 inventory, finding 1) > - The durable ownership plan (section A) asks for one owner stamp and one prover that returns `gone | alive | unprovable` > - This pull request adds that stamp and prover. Nothing calls it yet; the next PR uses it in a report-only reconciler > - The benefit is one tested definition of "owner gone" that later phases move each record kind onto ## Linked Issues or Issue Description Refs #19831 (plan, section A and Phase 1). Refs #19846 (Phase 0 inventory). Refs #19848 (Track 1: 1 s start-time gap and EPERM as alive). Stacked on #19851. **Problem** Each subsystem proves "owner gone" with its own rules. Some accept a dead PID on another host as proof. None returns "unprovable". **Expected behavior** One prover with one set of rules. A different host and an expired fence alone are never proof. "Unprovable" never releases a record. **Impact** This is the primitive that Phase 1's report-only reconciler and Phase 2's migrations use. ## What Changed - New `server/src/services/owner-proof.ts`: - `OwnerStamp`: a local stamp (`host`, `processSpace`, `bootId`, `pid`, `processStartedAt`, `processGroupId`, `generation`, `scope`) or a remote stamp (`fenceToken`, `generation`, `scope`, `provider`). - `proveOwnerGone(stamp, …)` returns `{ verdict: "gone" | "alive" | "unprovable", reason }`. - Local rules: this boot is alive; the stamp must name this host and PID space (host, kernel boot, PID namespace); the PID must be dead or reused (start-time gap of at least 1 s, via Track 1's `processStartsDefinitelyDiffer`); EPERM is alive; the process group must be empty or led by a newer process. - Remote rules: a superseded fence token or generation is gone; a provider stop proof is gone; a held fence is alive; a matching fence that only expired is unprovable. Process liveness is never consulted for a remote owner. - `currentLocalOwnerStamp()` reads only cached identity, so writing a stamp adds no OS call to the turn path. - `controller-process-identity.ts` exports `observedProcessStartedAt` and `UNKNOWN_PROCESS_SPACE` for the prover. - New `owner-proof.test.ts`: the proof matrix (EPERM, ESRCH, PID reuse inside and at 1 s, unreadable start time, a different host, a different or unknown PID space, process groups, a held, superseded, expired or provider-proved remote fence). ## Verification - `npx vitest run src/services/owner-proof.test.ts` (in `server/`): 20 tests pass. - `npx tsc --noEmit -p server`: no errors. ## Risks - Low risk. No caller uses the prover yet, so no behavior changes. - Previous-release story (`rules/upgrade-safe-run-state.mdc`): no persisted shape changes. Stamps from older rows that lack fields return `unprovable` (`missing_identity`), which never releases a record. - Track 1 is still fixing proofs in place. The prover reuses Track 1's helpers (`processExists`, `processStartsDefinitelyDiffer`), so a later Track 1 rule change applies here too. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, in Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../services/controller-process-identity.ts | 11 +- server/src/services/owner-proof.test.ts | 221 +++++++++++++++ server/src/services/owner-proof.ts | 254 ++++++++++++++++++ 3 files changed, 482 insertions(+), 4 deletions(-) create mode 100644 server/src/services/owner-proof.test.ts create mode 100644 server/src/services/owner-proof.ts diff --git a/server/src/services/controller-process-identity.ts b/server/src/services/controller-process-identity.ts index 29500709a7b..deec5a00db6 100644 --- a/server/src/services/controller-process-identity.ts +++ b/server/src/services/controller-process-identity.ts @@ -20,7 +20,7 @@ export type ControllerProcess = { bootId: string; pid: number; hostHash: string; * from any earlier boot never did. */ const bootId = randomUUID(); -const UNKNOWN_PROCESS_SPACE = "unknown"; +export const UNKNOWN_PROCESS_SPACE = "unknown"; /** * Names the space in which PIDs are comparable: one kernel boot and, on Linux, @@ -48,7 +48,8 @@ function processSpaceHash(): string { const hostHash = processSpaceHash(); -async function observedProcessStartedAt(pid: number): Promise { +/** OS-observed start time of any PID (epoch ms), or null when it cannot be read. */ +export async function observedProcessStartedAt(pid: number): Promise { try { const startedAt = Date.parse(await readProcessStartedAt(pid) ?? ""); return Number.isFinite(startedAt) ? startedAt : null; @@ -62,6 +63,8 @@ async function observedProcessStartedAt(pid: number): Promise { // claim, preparation check and lease renewal. A failed read is not cached, so // a transient failure is retried by the next caller. let ownStartedAt: Promise | undefined; +/** The start time from the last resolved read, for `currentControllerProcessStamp`. */ +let resolvedProcessStartedAt: string | null = null; /** OS-observed start time of this server process (epoch ms), read at most once * successfully per process. Null when the OS cannot report it. */ @@ -71,6 +74,8 @@ export function currentProcessStartedAtMs(): Promise { ownStartedAt = read; void read.then(value => { if (value === null && ownStartedAt === read) ownStartedAt = undefined; + // Any resolved read also feeds the no-I/O stamp. + if (value !== null) resolvedProcessStartedAt = new Date(value).toISOString(); }); } return ownStartedAt; @@ -85,8 +90,6 @@ export function currentControllerBootId(): string { return bootId; } -let resolvedProcessStartedAt: string | null = null; - export async function currentControllerProcess(): Promise { const startedAt = await currentProcessStartedAtMs(); const iso = startedAt === null ? null : new Date(startedAt).toISOString(); diff --git a/server/src/services/owner-proof.test.ts b/server/src/services/owner-proof.test.ts new file mode 100644 index 00000000000..d3ce7eb88d7 --- /dev/null +++ b/server/src/services/owner-proof.test.ts @@ -0,0 +1,221 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { currentProcessStartedAtMs, processExists, resetCurrentProcessStartedAtForTests } from "./controller-process-identity.js"; +import * as hotRestart from "./hot-restart.js"; +import { + currentLocalOwnerStamp, + currentProcessProbe, + proveOwnerGone, + type LocalOwnerStamp, + type ProcessProbe, + type RemoteOwnerStamp, +} from "./owner-proof.js"; + +const STARTED = "2026-11-07T10:00:00.000Z"; +const at = (offsetMs: number) => new Date(Date.parse(STARTED) + offsetMs); + +function stamp(overrides: Partial = {}): LocalOwnerStamp { + return { + kind: "local", scope: "test", generation: 1, host: "host-a", processSpace: "space-a", + bootId: "previous-boot", pid: 4242, processStartedAt: STARTED, processGroupId: null, ...overrides, + }; +} + +function probe(overrides: Partial & { + alive?: number[]; groups?: number[]; starts?: Record; +} = {}): ProcessProbe { + const { alive = [], groups = [], starts = {}, ...rest } = overrides; + return { + bootId: "current-boot", host: "host-a", processSpace: "space-a", + pidExists: pid => alive.includes(pid), + processGroupExists: group => groups.includes(group), + readStartedAt: async pid => starts[pid] ?? null, + ...rest, + }; +} + +afterEach(() => vi.restoreAllMocks()); + +describe("proveOwnerGone: local owners", () => { + it("treats this process's own boot as alive", async () => { + expect(await proveOwnerGone(stamp({ bootId: "current-boot" }), probe())).toEqual({ verdict: "alive", reason: "current_process" }); + }); + + it("proves a dead PID in the same PID space gone", async () => { + expect(await proveOwnerGone(stamp(), probe())).toEqual({ verdict: "gone", reason: "pid_dead" }); + }); + + it("never treats a different host as proof, even when the PID is dead here", async () => { + expect(await proveOwnerGone(stamp({ host: "host-b" }), probe())).toEqual({ verdict: "unprovable", reason: "different_host" }); + }); + + it("never treats a different PID namespace or kernel boot as proof", async () => { + expect(await proveOwnerGone(stamp({ processSpace: "space-b" }), probe())) + .toEqual({ verdict: "unprovable", reason: "different_process_space" }); + expect(await proveOwnerGone(stamp({ processSpace: "unknown" }), probe())) + .toEqual({ verdict: "unprovable", reason: "unknown_process_space" }); + expect(await proveOwnerGone(stamp(), probe({ processSpace: "unknown" }))) + .toEqual({ verdict: "unprovable", reason: "unknown_process_space" }); + }); + + it("treats a stamp without a PID or process space as unprovable", async () => { + expect(await proveOwnerGone(stamp({ pid: null }), probe())).toEqual({ verdict: "unprovable", reason: "missing_identity" }); + expect(await proveOwnerGone(stamp({ processSpace: null }), probe())).toEqual({ verdict: "unprovable", reason: "missing_identity" }); + }); + + it("keeps a live PID whose start time is within 1 s of the stamp alive (ps lstart precision)", async () => { + for (const offset of [0, 1, 999, -999]) { + expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(offset) } }))) + .toEqual({ verdict: "alive", reason: "pid_alive" }); + } + }); + + it("proves a reused PID gone only at a start-time gap of at least 1 s", async () => { + expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(1_000) } }))) + .toEqual({ verdict: "gone", reason: "pid_reused" }); + expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(60_000) } }))) + .toEqual({ verdict: "gone", reason: "pid_reused" }); + }); + + it("treats a live PID that started before the stamp as inconsistent, not reused", async () => { + expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(-60_000) } }))) + .toEqual({ verdict: "unprovable", reason: "pid_start_inconsistent" }); + }); + + it("treats a live PID with an unreadable or unrecorded start time as unprovable", async () => { + expect(await proveOwnerGone(stamp(), probe({ alive: [4242] }))) + .toEqual({ verdict: "unprovable", reason: "pid_start_unreadable" }); + expect(await proveOwnerGone(stamp({ processStartedAt: null }), probe({ alive: [4242], starts: { 4242: at(5_000) } }))) + .toEqual({ verdict: "unprovable", reason: "pid_start_unreadable" }); + }); + + it("treats EPERM from signal 0 as a live process", async () => { + vi.spyOn(process, "kill").mockImplementation(() => { + throw Object.assign(new Error("operation not permitted"), { code: "EPERM" }); + }); + expect(processExists(4242)).toBe(true); + const real = currentProcessProbe(); + const result = await proveOwnerGone( + stamp({ host: real.host, processSpace: real.processSpace, processStartedAt: STARTED }), + { ...real, readStartedAt: async () => at(0) }, + ); + expect(result).toEqual({ verdict: "alive", reason: "pid_alive" }); + }); + + it("treats ESRCH from signal 0 as a dead process", async () => { + vi.spyOn(process, "kill").mockImplementation(() => { + throw Object.assign(new Error("no such process"), { code: "ESRCH" }); + }); + expect(processExists(4242)).toBe(false); + }); + + it("keeps the owner alive while its own process group still has members", async () => { + // The leader exited; its children keep the group alive. + expect(await proveOwnerGone(stamp({ processGroupId: 4242 }), probe({ groups: [4242] }))) + .toEqual({ verdict: "alive", reason: "process_group_alive" }); + // The recorded leader still leads the group. + expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: STARTED }), + probe({ groups: [5000], alive: [5000], starts: { 5000: at(500) } }))) + .toEqual({ verdict: "alive", reason: "process_group_alive" }); + }); + + it("never proves gone a surviving child group that started after its parent", async () => { + // The owner started at 10:00, spawned a detached group at 10:01 and exited. + // Without the group leader's recorded start time, the group may be its own. + expect(await proveOwnerGone(stamp({ processGroupId: 5000 }), + probe({ groups: [5000], alive: [5000], starts: { 5000: at(60_000) } }))) + .toEqual({ verdict: "unprovable", reason: "process_group_ambiguous" }); + expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: at(60_000).toISOString() }), + probe({ groups: [5000], alive: [5000], starts: { 5000: at(60_000) } }))) + .toEqual({ verdict: "alive", reason: "process_group_alive" }); + }); + + it("proves the owner gone when its process group is empty or led by a different process", async () => { + expect(await proveOwnerGone(stamp({ processGroupId: 4242 }), probe())).toEqual({ verdict: "gone", reason: "pid_dead" }); + // A reused owner PID now leads group 4242: the kernel does not reuse a PID + // while its old group has members, so the old group is empty. + expect(await proveOwnerGone(stamp({ processGroupId: 4242 }), + probe({ groups: [4242], alive: [4242], starts: { 4242: at(30_000) } }))) + .toEqual({ verdict: "gone", reason: "pid_reused" }); + expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: at(-10_000).toISOString() }), + probe({ groups: [5000], alive: [5000], starts: { 5000: at(30_000) } }))) + .toEqual({ verdict: "gone", reason: "pid_dead" }); + }); + + it("never treats an older group leader as a replacement group", async () => { + expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: STARTED }), + probe({ groups: [5000], alive: [5000], starts: { 5000: at(-30_000) } }))) + .toEqual({ verdict: "unprovable", reason: "process_group_ambiguous" }); + }); + + it("treats a non-empty group with an unreadable leader as unprovable", async () => { + expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: STARTED }), + probe({ groups: [5000], alive: [5000] }))) + .toEqual({ verdict: "unprovable", reason: "process_group_ambiguous" }); + }); + + it("stamps the start time resolved by any earlier identity read", async () => { + resetCurrentProcessStartedAtForTests(); + vi.spyOn(hotRestart, "readProcessStartedAt").mockResolvedValue("2026-11-07T09:00:00.000Z"); + await currentProcessStartedAtMs(); + expect(currentLocalOwnerStamp("test").processStartedAt).toBe("2026-11-07T09:00:00.000Z"); + }); + + it("stamps this process without I/O and proves it alive", async () => { + const read = vi.spyOn(hotRestart, "readProcessStartedAt"); + const own = currentLocalOwnerStamp("test", 3); + expect(own).toMatchObject({ kind: "local", pid: process.pid, generation: 3 }); + expect(read).not.toHaveBeenCalled(); + expect(await proveOwnerGone(own)).toEqual({ verdict: "alive", reason: "current_process" }); + }); +}); + +describe("proveOwnerGone: remote owners (fenced handoff)", () => { + const remote = (overrides: Partial = {}): RemoteOwnerStamp => ({ + kind: "remote", scope: "test", generation: 2, fenceToken: "boot:2:token", provider: "daytona", ...overrides, + }); + const now = new Date("2026-11-07T12:00:00.000Z"); + const later = new Date(now.getTime() + 30_000); + const earlier = new Date(now.getTime() - 30_000); + + it("keeps a held fence alive", async () => { + expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: later }, now)) + .toEqual({ verdict: "alive", reason: "fence_held" }); + }); + + it("proves a superseded fence token or generation gone", async () => { + expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:3:other", currentGeneration: 2, leaseExpiresAt: later }, now)) + .toEqual({ verdict: "gone", reason: "fence_superseded" }); + expect(await proveOwnerGone(remote(), { currentFenceToken: null, currentGeneration: 2, leaseExpiresAt: null }, now)) + .toEqual({ verdict: "gone", reason: "fence_superseded" }); + expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 3, leaseExpiresAt: later }, now)) + .toEqual({ verdict: "gone", reason: "generation_superseded" }); + }); + + it("never proves an expired fence gone by age alone", async () => { + expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier }, now)) + .toEqual({ verdict: "unprovable", reason: "fence_expired_without_provider_proof" }); + }); + + it("uses the provider proof where one exists", async () => { + expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier, provider: "stopped" }, now)) + .toEqual({ verdict: "gone", reason: "provider_stopped" }); + expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier, provider: "running" }, now)) + .toEqual({ verdict: "alive", reason: "provider_running" }); + }); + + it("does not consult process liveness for a remote owner", async () => { + const kill = vi.spyOn(process, "kill"); + await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier }, now); + expect(kill).not.toHaveBeenCalled(); + }); + + it("never ties a provider stop proof to a stamp without a fence token", async () => { + expect(await proveOwnerGone(remote({ fenceToken: null }), { currentFenceToken: "boot:2:token", currentGeneration: 2, + leaseExpiresAt: earlier, provider: "stopped" }, now)).toEqual({ verdict: "unprovable", reason: "missing_identity" }); + }); + + it("treats a remote stamp without a fence token as unprovable", async () => { + expect(await proveOwnerGone(remote({ fenceToken: null }), { currentFenceToken: null, currentGeneration: null, leaseExpiresAt: null }, now)) + .toEqual({ verdict: "unprovable", reason: "missing_identity" }); + }); +}); diff --git a/server/src/services/owner-proof.ts b/server/src/services/owner-proof.ts new file mode 100644 index 00000000000..3093c8dd2dc --- /dev/null +++ b/server/src/services/owner-proof.ts @@ -0,0 +1,254 @@ +import { hostname } from "node:os"; +import { + currentControllerBootId, + currentControllerProcessStamp, + currentProcessSpaceHash, + observedProcessStartedAt, + processExists, + processStartsDefinitelyDiffer, + PROCESS_START_PRECISION_MS, + UNKNOWN_PROCESS_SPACE, +} from "./controller-process-identity.js"; + +/** + * The owner recorded on an in-progress durable record (plan section A). + * + * A local owner is a process in a PID space. A remote owner is a fenced lease: + * ownership moves by fence token and generation, never by a process proof, + * because a remote runner keeps running across a control-plane replacement. + */ +export type OwnerStamp = LocalOwnerStamp | RemoteOwnerStamp; + +export type LocalOwnerStamp = { + kind: "local"; + /** Record kind or subsystem that wrote the stamp, for reports only. */ + scope: string; + generation: number | null; + /** Hostname at stamp time. Informational; `processSpace` is the proof key. */ + host: string | null; + /** Hash of host, kernel boot and PID namespace (`currentProcessSpaceHash`). */ + processSpace: string | null; + bootId: string | null; + pid: number | null; + processStartedAt: string | null; + processGroupId?: number | null; + /** Start time of the process group's leader (PID = `processGroupId`). When + * the owner leads its own group this is `processStartedAt`. */ + processGroupStartedAt?: string | null; +}; + +export type RemoteOwnerStamp = { + kind: "remote"; + scope: string; + generation: number | null; + /** The fence or lease token the owner holds, such as `native_run_finalizations.lease_owner`. */ + fenceToken: string | null; + provider?: string | null; +}; + +export type OwnerVerdict = "gone" | "alive" | "unprovable"; + +export type OwnerProof = { verdict: OwnerVerdict; reason: OwnerProofReason }; + +export type OwnerProofReason = + // gone + | "pid_dead" + | "pid_reused" + | "fence_superseded" + | "generation_superseded" + | "provider_stopped" + // alive + | "current_process" + | "pid_alive" + | "process_group_alive" + | "fence_held" + | "provider_running" + // unprovable + | "missing_identity" + | "unknown_process_space" + | "different_process_space" + | "different_host" + | "pid_start_unreadable" + | "pid_start_inconsistent" + | "process_group_ambiguous" + | "fence_expired_without_provider_proof"; + +/** The state a remote owner is judged against, read by the caller from the + * durable record that holds the fence. */ +export type RemoteFenceState = { + /** The token that currently holds the fence, or null when released. */ + currentFenceToken: string | null; + currentGeneration: number | null; + /** When the current fence expires. Expiry alone never proves an owner gone. */ + leaseExpiresAt: Date | null; + /** A provider proof for the stamped lease, when the provider gives one + * (for example a remote termination receipt). */ + provider?: "stopped" | "running" | "unknown"; +}; + +/** OS access, injectable so the proof matrix is testable without real PIDs. */ +export type ProcessProbe = { + bootId: string; + host: string; + processSpace: string; + pidExists(pid: number): boolean; + processGroupExists(processGroupId: number): boolean; + readStartedAt(pid: number): Promise; +}; + +function processGroupExists(processGroupId: number): boolean { + if (process.platform === "win32") return false; + try { + process.kill(-processGroupId, 0); + return true; + } catch (error) { + // EPERM: the group has members owned by another user. Alive. + return (error as NodeJS.ErrnoException | undefined)?.code === "EPERM"; + } +} + +export function currentProcessProbe(): ProcessProbe { + return { + bootId: currentControllerBootId(), + host: hostname(), + processSpace: currentProcessSpaceHash(), + pidExists: processExists, + processGroupExists, + readStartedAt: async pid => { + const startedAt = await observedProcessStartedAt(pid); + return startedAt === null ? null : new Date(startedAt); + }, + }; +} + +/** The stamp this process writes when it takes ownership of a record. It does + * no I/O: the start time comes from an earlier resolved identity read, and is + * null until one has happened. A stamp without a start time still proves exit + * when its PID is gone; a live PID then stays unprovable, which fails closed. + * + * It records no process group: the control-plane server owns the record, and + * its provider children are tracked by their own records (the run's process + * identity, the runner's startup attempt). A caller that stamps a child it + * leads must set `processGroupId` and `processGroupStartedAt` itself; without + * them a surviving group is never judged at all. */ +export function currentLocalOwnerStamp(scope: string, generation: number | null = null): LocalOwnerStamp { + const controller = currentControllerProcessStamp(); + return { + kind: "local", scope, generation, host: hostname(), processSpace: controller.hostHash, + bootId: controller.bootId, pid: controller.pid, processStartedAt: controller.processStartedAt, + processGroupId: null, + }; +} + +const validPid = (pid: number | null | undefined): pid is number => + typeof pid === "number" && Number.isInteger(pid) && pid > 0; + +/** + * Proves whether the owner in a stamp is gone. Proof is by identity, never by + * age. `unprovable` must never release a record. + * + * Local rules (one prover for every kind): + * - this process's boot id is alive; + * - the stamp must name this PID space (host, kernel boot, PID namespace); a + * different host or namespace is never proof; + * - the PID must be dead, or reused: a live PID that started at least 1 s + * after the stamp (`ps lstart` has 1 s precision). An earlier start is + * inconsistent and unprovable. EPERM is alive; + * - the recorded process group must be empty, or led by a process that + * started definitely after the recorded group leader. A non-empty group without a + * recorded leader start time is unprovable: it may hold surviving children. + * + * Remote rules: a superseded fence token or generation, or a provider proof + * that the stamped lease stopped, is gone. A matching live fence is alive. A + * matching fence that only expired, with no provider proof, is unprovable. + */ +export async function proveOwnerGone( + stamp: LocalOwnerStamp, + probe?: ProcessProbe, +): Promise; +export async function proveOwnerGone( + stamp: RemoteOwnerStamp, + fence: RemoteFenceState, + now?: Date, +): Promise; +export async function proveOwnerGone( + stamp: OwnerStamp, + context?: ProcessProbe | RemoteFenceState, + now: Date = new Date(), +): Promise { + if (stamp.kind === "remote") return proveRemoteOwnerGone(stamp, context as RemoteFenceState, now); + return proveLocalOwnerGone(stamp, (context as ProcessProbe | undefined) ?? currentProcessProbe()); +} + +async function proveLocalOwnerGone(stamp: LocalOwnerStamp, probe: ProcessProbe): Promise { + if (stamp.bootId && stamp.bootId === probe.bootId) return { verdict: "alive", reason: "current_process" }; + if (!validPid(stamp.pid) || !stamp.processSpace) return { verdict: "unprovable", reason: "missing_identity" }; + if (stamp.processSpace === UNKNOWN_PROCESS_SPACE || probe.processSpace === UNKNOWN_PROCESS_SPACE) { + return { verdict: "unprovable", reason: "unknown_process_space" }; + } + if (stamp.host && stamp.host !== probe.host) return { verdict: "unprovable", reason: "different_host" }; + if (stamp.processSpace !== probe.processSpace) return { verdict: "unprovable", reason: "different_process_space" }; + + let processReason: "pid_dead" | "pid_reused"; + if (!probe.pidExists(stamp.pid)) { + processReason = "pid_dead"; + } else { + const observed = await probe.readStartedAt(stamp.pid); + if (!stamp.processStartedAt || observed === null) { + return { verdict: "unprovable", reason: "pid_start_unreadable" }; + } + const order = compareStarts(stamp.processStartedAt, observed); + if (order === "same") return { verdict: "alive", reason: "pid_alive" }; + // A reused PID belongs to a later process. An earlier start means the + // stamp is inconsistent (clock skew or a migrated value): never proof. + if (order !== "later") return { verdict: "unprovable", reason: "pid_start_inconsistent" }; + processReason = "pid_reused"; + } + + const group = stamp.processGroupId; + if (validPid(group) && probe.processGroupExists(group)) { + // A non-empty group can hold the owner's surviving children, including + // detached ones started after the owner. The kernel never reuses a PID + // while a group with that id has members, so the group is a different one + // only when its leader is alive and is definitely not the recorded leader. + if (!probe.pidExists(group)) return { verdict: "alive", reason: "process_group_alive" }; + const recordedLeaderStart = group === stamp.pid ? stamp.processStartedAt : stamp.processGroupStartedAt ?? null; + const leaderStartedAt = await probe.readStartedAt(group); + if (!recordedLeaderStart || leaderStartedAt === null) { + return { verdict: "unprovable", reason: "process_group_ambiguous" }; + } + const order = compareStarts(recordedLeaderStart, leaderStartedAt); + if (order === "same") return { verdict: "alive", reason: "process_group_alive" }; + if (order !== "later") return { verdict: "unprovable", reason: "process_group_ambiguous" }; + } + return { verdict: "gone", reason: processReason }; +} + +/** "same" within `ps lstart` precision (1 s), "later" when the observed process + * started definitely after the recorded one, "earlier" when definitely before, + * "unknown" when either time is unreadable. */ +function compareStarts(recorded: string, observed: Date): "same" | "later" | "earlier" | "unknown" { + const left = Date.parse(recorded); + const right = observed.getTime(); + if (!Number.isFinite(left) || !Number.isFinite(right)) return "unknown"; + if (!processStartsDefinitelyDiffer(recorded, observed)) return "same"; + return right - left >= PROCESS_START_PRECISION_MS ? "later" : "earlier"; +} + +function proveRemoteOwnerGone(stamp: RemoteOwnerStamp, fence: RemoteFenceState, now: Date): OwnerProof { + // The identity check comes first: a provider proof for the current lease + // cannot be tied to a stamp that names no token. + if (!stamp.fenceToken) return { verdict: "unprovable", reason: "missing_identity" }; + if (fence.provider === "stopped") return { verdict: "gone", reason: "provider_stopped" }; + if (stamp.generation !== null && fence.currentGeneration !== null && fence.currentGeneration > stamp.generation) { + return { verdict: "gone", reason: "generation_superseded" }; + } + if (fence.currentFenceToken !== stamp.fenceToken) { + // Released (null) or handed to another owner: the stamped owner no longer + // holds authority, whether or not its process still runs. + return { verdict: "gone", reason: "fence_superseded" }; + } + if (fence.provider === "running") return { verdict: "alive", reason: "provider_running" }; + if (fence.leaseExpiresAt && fence.leaseExpiresAt > now) return { verdict: "alive", reason: "fence_held" }; + return { verdict: "unprovable", reason: "fence_expired_without_provider_proof" }; +}