import { afterEach, describe, expect, it, vi } from "vitest"; import { currentProcessStartedAtMs, processExists, resetCurrentProcessStartedAtForTests } from "./controller-process-identity.js"; import * as hotRestart from "./hot-restart.js"; import { currentLocalOwnerStamp, currentDeploymentIdentity, currentProcessProbe, proveOwnerGone, type LocalOwnerStamp, type ProcessProbe, type RemoteOwnerStamp, } from "./owner-proof.js"; const STARTED = "2026-11-07T10:00:00.000Z"; const at = (offsetMs: number) => new Date(Date.parse(STARTED) + offsetMs); function stamp(overrides: Partial = {}): LocalOwnerStamp { return { kind: "local", scope: "test", generation: 1, host: "host-a", processSpace: "space-a", bootId: "previous-boot", pid: 4242, processStartedAt: STARTED, processGroupId: null, ...overrides, }; } function probe(overrides: Partial & { alive?: number[]; groups?: number[]; starts?: Record; } = {}): ProcessProbe { const { alive = [], groups = [], starts = {}, ...rest } = overrides; return { bootId: "current-boot", host: "host-a", processSpace: "space-a", pidExists: pid => alive.includes(pid), processGroupExists: group => groups.includes(group), readStartedAt: async pid => starts[pid] ?? null, ...rest, }; } afterEach(() => vi.restoreAllMocks()); describe("proveOwnerGone: local owners", () => { it("treats this process's own boot as alive", async () => { expect(await proveOwnerGone(stamp({ bootId: "current-boot" }), probe())).toEqual({ verdict: "alive", reason: "current_process" }); }); it("proves a dead PID in the same PID space gone", async () => { expect(await proveOwnerGone(stamp(), probe())).toEqual({ verdict: "gone", reason: "pid_dead" }); }); it.each([null, "foreign-space", "unknown"])("does not probe a recorded-space-required owner in %s space", async processSpace => { const pidExists = vi.fn(() => false), readStartedAt = vi.fn(async () => null); const result = await proveOwnerGone(stamp({ processSpace, processSpaceUnrecorded: processSpace === null, requireRecordedProcessSpace: true }), probe({ pidExists, readStartedAt })); expect(result).toEqual({ verdict: "unprovable", reason: processSpace === null ? "process_space_unrecorded" : processSpace === "unknown" ? "unknown_process_space" : "different_process_space" }); expect(pidExists).not.toHaveBeenCalled(); expect(readStartedAt).not.toHaveBeenCalled(); }); it.each(["stopped", "fenced"] as const)("retains independent controller-instance %s evidence for an unrecorded Native space", async state => { const pidExists = vi.fn(() => false), readStartedAt = vi.fn(async () => null); const controllerInstance = vi.fn(async () => state); expect(await proveOwnerGone(stamp({ processSpace: null, processSpaceUnrecorded: true, requireRecordedProcessSpace: true }), probe({ pidExists, readStartedAt, controllerInstance }))) .toEqual({ verdict: "gone", reason: state === "stopped" ? "controller_stopped" : "controller_fenced" }); expect(controllerInstance).toHaveBeenCalledOnce(); expect(pidExists).not.toHaveBeenCalled(); expect(readStartedAt).not.toHaveBeenCalled(); }); it("accepts audited operator stop evidence without probing a recorded-space-required owner", async () => { const pidExists = vi.fn(() => false), readStartedAt = vi.fn(async () => null); expect(await proveOwnerGone(stamp({ processSpace: "operator_attested", requireRecordedProcessSpace: true }), probe({ pidExists, readStartedAt }))).toEqual({ verdict: "gone", reason: "operator_attested" }); expect(pidExists).not.toHaveBeenCalled(); expect(readStartedAt).not.toHaveBeenCalled(); }); it("judges a row saved before its PID space locally for one release, on single-host servers only", async () => { const legacy = stamp({ host: null, processSpace: null, processSpaceUnrecorded: true }); expect(await proveOwnerGone(legacy, probe())).toEqual({ verdict: "gone", reason: "pid_dead", legacyHostFallback: true }); expect(await proveOwnerGone(legacy, probe({ alive: [4242], starts: { 4242: at(0) } }))) .toEqual({ verdict: "alive", reason: "pid_alive" }); // Without the marker a missing space is missing identity, never proof. expect(await proveOwnerGone(stamp({ processSpace: null }), probe())).toEqual({ verdict: "unprovable", reason: "missing_identity" }); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); try { expect(await proveOwnerGone(legacy, probe())).toEqual({ verdict: "unprovable", reason: "process_space_unrecorded" }); } finally { vi.unstubAllEnvs(); } }); it("accepts a board attestation that this exact recorded owner stopped", async () => { expect(await proveOwnerGone(stamp({ processSpace: "operator_attested" }), probe({ alive: [4242], starts: { 4242: at(0) } }))) .toEqual({ verdict: "gone", reason: "operator_attested" }); }); it("never treats a different host as proof, even when the PID is dead here", async () => { expect(await proveOwnerGone(stamp({ host: "host-b" }), probe())).toEqual({ verdict: "unprovable", reason: "different_host" }); }); it("never treats a different PID namespace or kernel boot as proof", async () => { expect(await proveOwnerGone(stamp({ processSpace: "space-b" }), probe())) .toEqual({ verdict: "unprovable", reason: "different_process_space" }); expect(await proveOwnerGone(stamp({ processSpace: "unknown" }), probe())) .toEqual({ verdict: "unprovable", reason: "unknown_process_space" }); expect(await proveOwnerGone(stamp(), probe({ processSpace: "unknown" }))) .toEqual({ verdict: "unprovable", reason: "unknown_process_space" }); }); it("treats a stamp without a PID or process space as unprovable", async () => { expect(await proveOwnerGone(stamp({ pid: null }), probe())).toEqual({ verdict: "unprovable", reason: "missing_identity" }); expect(await proveOwnerGone(stamp({ processSpace: null }), probe())).toEqual({ verdict: "unprovable", reason: "missing_identity" }); }); it("keeps a live PID whose start time is within 1 s of the stamp alive (ps lstart precision)", async () => { for (const offset of [0, 1, 999, -999]) { expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(offset) } }))) .toEqual({ verdict: "alive", reason: "pid_alive" }); } }); it("proves a reused PID gone only at a start-time gap of at least 1 s", async () => { expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(1_000) } }))) .toEqual({ verdict: "gone", reason: "pid_reused" }); expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(60_000) } }))) .toEqual({ verdict: "gone", reason: "pid_reused" }); }); it("treats a live PID that started before the stamp as inconsistent, not reused", async () => { expect(await proveOwnerGone(stamp(), probe({ alive: [4242], starts: { 4242: at(-60_000) } }))) .toEqual({ verdict: "unprovable", reason: "pid_start_inconsistent" }); }); it("treats a live PID with an unreadable or unrecorded start time as unprovable", async () => { expect(await proveOwnerGone(stamp(), probe({ alive: [4242] }))) .toEqual({ verdict: "unprovable", reason: "pid_start_unreadable" }); expect(await proveOwnerGone(stamp({ processStartedAt: null }), probe({ alive: [4242], starts: { 4242: at(5_000) } }))) .toEqual({ verdict: "unprovable", reason: "pid_start_unreadable" }); }); it("treats EPERM from signal 0 as a live process", async () => { vi.spyOn(process, "kill").mockImplementation(() => { throw Object.assign(new Error("operation not permitted"), { code: "EPERM" }); }); expect(processExists(4242)).toBe(true); const real = currentProcessProbe(); const result = await proveOwnerGone( stamp({ host: real.host, processSpace: real.processSpace, processStartedAt: STARTED }), { ...real, readStartedAt: async () => at(0) }, ); expect(result).toEqual({ verdict: "alive", reason: "pid_alive" }); }); it("treats ESRCH from signal 0 as a dead process", async () => { vi.spyOn(process, "kill").mockImplementation(() => { throw Object.assign(new Error("no such process"), { code: "ESRCH" }); }); expect(processExists(4242)).toBe(false); }); it("keeps the owner alive while its own process group still has members", async () => { // The leader exited; its children keep the group alive. expect(await proveOwnerGone(stamp({ processGroupId: 4242 }), probe({ groups: [4242] }))) .toEqual({ verdict: "alive", reason: "process_group_alive" }); // The recorded leader still leads the group. expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: STARTED }), probe({ groups: [5000], alive: [5000], starts: { 5000: at(500) } }))) .toEqual({ verdict: "alive", reason: "process_group_alive" }); }); it("never proves gone a surviving child group that started after its parent", async () => { // The owner started at 10:00, spawned a detached group at 10:01 and exited. // Without the group leader's recorded start time, the group may be its own. expect(await proveOwnerGone(stamp({ processGroupId: 5000 }), probe({ groups: [5000], alive: [5000], starts: { 5000: at(60_000) } }))) .toEqual({ verdict: "unprovable", reason: "process_group_ambiguous" }); expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: at(60_000).toISOString() }), probe({ groups: [5000], alive: [5000], starts: { 5000: at(60_000) } }))) .toEqual({ verdict: "alive", reason: "process_group_alive" }); }); it("proves the owner gone when its process group is empty or led by a different process", async () => { expect(await proveOwnerGone(stamp({ processGroupId: 4242 }), probe())).toEqual({ verdict: "gone", reason: "pid_dead" }); // A reused owner PID now leads group 4242: the kernel does not reuse a PID // while its old group has members, so the old group is empty. expect(await proveOwnerGone(stamp({ processGroupId: 4242 }), probe({ groups: [4242], alive: [4242], starts: { 4242: at(30_000) } }))) .toEqual({ verdict: "gone", reason: "pid_reused" }); expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: at(-10_000).toISOString() }), probe({ groups: [5000], alive: [5000], starts: { 5000: at(30_000) } }))) .toEqual({ verdict: "gone", reason: "pid_dead" }); }); it("never treats an older group leader as a replacement group", async () => { expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: STARTED }), probe({ groups: [5000], alive: [5000], starts: { 5000: at(-30_000) } }))) .toEqual({ verdict: "unprovable", reason: "process_group_ambiguous" }); }); it("treats a non-empty group with an unreadable leader as unprovable", async () => { expect(await proveOwnerGone(stamp({ processGroupId: 5000, processGroupStartedAt: STARTED }), probe({ groups: [5000], alive: [5000] }))) .toEqual({ verdict: "unprovable", reason: "process_group_ambiguous" }); }); it("stamps the start time resolved by any earlier identity read", async () => { resetCurrentProcessStartedAtForTests(); vi.spyOn(hotRestart, "readProcessStartedAt").mockResolvedValue("2026-11-07T09:00:00.000Z"); await currentProcessStartedAtMs(); expect(currentLocalOwnerStamp("test").processStartedAt).toBe("2026-11-07T09:00:00.000Z"); }); it("stamps this process without I/O and proves it alive", async () => { const read = vi.spyOn(hotRestart, "readProcessStartedAt"); const own = currentLocalOwnerStamp("test", 3); expect(own).toMatchObject({ kind: "local", pid: process.pid, generation: 3 }); expect(read).not.toHaveBeenCalled(); expect(await proveOwnerGone(own)).toEqual({ verdict: "alive", reason: "current_process" }); }); }); describe("proveOwnerGone: replaced platform deployment", () => { const worker = { provider: "railway" as const, projectId: "project", environmentId: "env", serviceId: "service", deploymentId: "old" }; const current = (overrides: Partial = {}, exclusiveVolume = true) => probe({ host: "new-container", processSpace: "space-new", deployment: { worker: { ...worker, deploymentId: "new", ...overrides }, exclusiveVolume } }); it("proves an owner on an earlier deployment of the same volume-backed service gone, on any host", async () => { expect(await proveOwnerGone(stamp({ worker, host: "old-container", processSpace: null }), current())) .toEqual({ verdict: "gone", reason: "deployment_replaced" }); }); it("never treats this process's own boot as replaced", async () => { expect(await proveOwnerGone(stamp({ worker, bootId: "current-boot" }), current())) .toEqual({ verdict: "alive", reason: "current_process" }); }); it("is no proof without an exclusive volume, for the same deployment, or for another service or environment", async () => { const old = stamp({ worker, host: "old-container" }); expect((await proveOwnerGone(old, current({}, false))).verdict).toBe("unprovable"); expect((await proveOwnerGone(old, current({ deploymentId: "old" }))).verdict).toBe("unprovable"); expect((await proveOwnerGone(old, current({ serviceId: "other" }))).verdict).toBe("unprovable"); expect((await proveOwnerGone(old, current({ environmentId: "other" }))).verdict).toBe("unprovable"); expect((await proveOwnerGone(stamp({ host: "old-container" }), current())).verdict).toBe("unprovable"); }); it("reads the deployment and volume from the Railway environment", () => { const env = { RAILWAY_PROJECT_ID: "p", RAILWAY_ENVIRONMENT_ID: "e", RAILWAY_SERVICE_ID: "s", RAILWAY_DEPLOYMENT_ID: "d" }; expect(currentDeploymentIdentity(env)).toEqual({ worker: { provider: "railway", projectId: "p", environmentId: "e", serviceId: "s", deploymentId: "d" }, exclusiveVolume: false, volumeId: null }); expect(currentDeploymentIdentity({ ...env, RAILWAY_VOLUME_ID: "v", RAILWAY_VOLUME_MOUNT_PATH: "/data" })?.exclusiveVolume).toBe(true); expect(currentDeploymentIdentity({})).toBeNull(); // A half-configured volume is not exclusivity: fall back to other proof. expect(currentDeploymentIdentity({ ...env, RAILWAY_VOLUME_ID: "v" })?.exclusiveVolume).toBe(false); expect(currentDeploymentIdentity({ ...env, RAILWAY_VOLUME_MOUNT_PATH: "/data" })?.exclusiveVolume).toBe(false); }); it("binds a stamp that recorded its volume to that same volume", async () => { const withVolume = (volumeId: string) => probe({ host: "new-container", deployment: { worker: { ...worker, deploymentId: "new" }, exclusiveVolume: true, volumeId } }); expect(await proveOwnerGone(stamp({ worker, volumeId: "vol-a" }), withVolume("vol-a"))) .toEqual({ verdict: "gone", reason: "deployment_replaced" }); expect((await proveOwnerGone(stamp({ worker, host: "old", volumeId: "vol-a" }), withVolume("vol-b"))).verdict).toBe("unprovable"); // A stamp from a deployment that was not volume-exclusive never qualifies. expect((await proveOwnerGone(stamp({ worker, host: "old", exclusiveVolume: false }), withVolume("vol-a"))).verdict).toBe("unprovable"); }); }); describe("proveOwnerGone: remote owners (fenced handoff)", () => { const remote = (overrides: Partial = {}): RemoteOwnerStamp => ({ kind: "remote", scope: "test", generation: 2, fenceToken: "boot:2:token", provider: "daytona", ...overrides, }); const now = new Date("2026-11-07T12:00:00.000Z"); const later = new Date(now.getTime() + 30_000); const earlier = new Date(now.getTime() - 30_000); it("keeps a held fence alive", async () => { expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: later }, now)) .toEqual({ verdict: "alive", reason: "fence_held" }); }); it("proves a superseded fence token or generation gone", async () => { expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:3:other", currentGeneration: 2, leaseExpiresAt: later }, now)) .toEqual({ verdict: "gone", reason: "fence_superseded" }); expect(await proveOwnerGone(remote(), { currentFenceToken: null, currentGeneration: 2, leaseExpiresAt: null }, now)) .toEqual({ verdict: "gone", reason: "fence_superseded" }); expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 3, leaseExpiresAt: later }, now)) .toEqual({ verdict: "gone", reason: "generation_superseded" }); }); it("never proves an expired fence gone by age alone", async () => { expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier }, now)) .toEqual({ verdict: "unprovable", reason: "fence_expired_without_provider_proof" }); }); it("uses the provider proof where one exists", async () => { expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier, provider: "stopped" }, now)) .toEqual({ verdict: "gone", reason: "provider_stopped" }); expect(await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier, provider: "running" }, now)) .toEqual({ verdict: "alive", reason: "provider_running" }); }); it("does not consult process liveness for a remote owner", async () => { const kill = vi.spyOn(process, "kill"); await proveOwnerGone(remote(), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier }, now); expect(kill).not.toHaveBeenCalled(); }); it("never ties a provider stop proof to a stamp without a fence token", async () => { expect(await proveOwnerGone(remote({ fenceToken: null }), { currentFenceToken: "boot:2:token", currentGeneration: 2, leaseExpiresAt: earlier, provider: "stopped" }, now)).toEqual({ verdict: "unprovable", reason: "missing_identity" }); }); it("treats a remote stamp without a fence token as unprovable", async () => { expect(await proveOwnerGone(remote({ fenceToken: null }), { currentFenceToken: null, currentGeneration: null, leaseExpiresAt: null }, now)) .toEqual({ verdict: "unprovable", reason: "missing_identity" }); }); });