diff --git a/server/src/__tests__/environment-lease-owner-stamps.test.ts b/server/src/__tests__/environment-lease-owner-stamps.test.ts new file mode 100644 index 00000000000..7e494ec0569 --- /dev/null +++ b/server/src/__tests__/environment-lease-owner-stamps.test.ts @@ -0,0 +1,117 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, environmentLeases, environments } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { currentControllerProcessStamp, currentProcessSpaceHash } from "../services/controller-process-identity.js"; +import { ENVIRONMENT_LEASE_CONTROLLER_KEY } from "../services/environment-lease-controller.js"; +import { environmentService } from "../services/environments.js"; +import { createOwnershipReportState, runOwnershipReconciliationPass } from "../services/recovery/ownership-invariant-report.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +async function exitedPid(): Promise { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" }); + const pid = child.pid!; + await new Promise(resolve => { child.once("exit", () => resolve()); child.kill("SIGKILL"); }); + return pid; +} + +suite("environment lease owner stamps (login/probe lease, teardown claim)", () => { + let db!: ReturnType; + let temporary: Awaited> | null = null; + let companyId = ""; + let environmentId = ""; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-environment-lease-owner-stamps-"); + db = createDb(temporary.connectionString); + companyId = randomUUID(); + environmentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: "ELS", + requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: randomUUID(), companyId, name: "Coder", role: "engineer", status: "idle", + adapterType: "codex_local", adapterConfig: {}, runtimeConfig: {}, permissions: {} }); + await db.insert(environments).values({ id: environmentId, name: "Fake Sandbox", driver: "sandbox", + status: "active", config: { provider: "fake", image: "ubuntu:24.04" } }); + }, 30_000); + + afterEach(async () => { + await db.delete(environmentLeases); + }); + + afterAll(async () => { + await temporary?.cleanup(); + }); + + async function report(kind: string) { + const result = await runOwnershipReconciliationPass(db, { state: createOwnershipReportState(), record: false }); + return { kind: result.kinds.find(entry => entry.kind === kind)!, violations: result.violations.filter(entry => entry.kind === kind) }; + } + + async function claim(metadata: Record) { + const id = randomUUID(); + await db.insert(environmentLeases).values({ id, companyId, environmentId, status: "pending_cleanup", + provider: "fake", providerLeaseId: `sandbox://fake/${id}`, metadata: { pendingCleanupInFlight: true, ...metadata } }); + return id; + } + + describe("login/probe lease", () => { + it("is owned by the process whose write made it active", async () => { + await environmentService(db).acquireLease({ companyId, environmentId, provider: "fake", + providerLeaseId: "sandbox://fake/login" }); + const { kind, violations } = await report("environment_login_probe_lease"); + expect(kind).toMatchObject({ inProgress: 1, verdicts: { alive: 1 }, reasons: { current_process: 1 } }); + expect(violations).toEqual([]); + }); + + it("is proven gone when that process has exited", async () => { + await db.insert(environmentLeases).values({ companyId, environmentId, status: "active", metadata: { + [ENVIRONMENT_LEASE_CONTROLLER_KEY]: { bootId: "earlier-boot", pid: await exitedPid(), + hostHash: currentProcessSpaceHash(), processStartedAt: null } } }); + const { kind } = await report("environment_login_probe_lease"); + expect(kind).toMatchObject({ verdicts: { gone: 1 }, reasons: { pid_dead: 1 } }); + }); + + it("names no owner when the previous release saved it without a stamp", async () => { + await db.insert(environmentLeases).values({ companyId, environmentId, status: "active", metadata: {} }); + const { kind, violations } = await report("environment_login_probe_lease"); + expect(kind).toMatchObject({ verdicts: { unprovable: 1 }, reasons: { no_owner_stamp: 1 } }); + expect(violations.map(violation => violation.rule)).toEqual(["I1_owned"]); + }); + }); + + describe("teardown claim", () => { + it("is owned by the claimant process stamped for this exact attempt", async () => { + const attemptId = randomUUID(); + await claim({ pendingCleanupAttemptId: attemptId, pendingCleanupLeaseExpiresAtMs: Date.now() + 60_000, + pendingCleanupController: { bootId: "earlier-boot", pid: await exitedPid(), hostHash: currentProcessSpaceHash(), + processStartedAt: null, attemptId } }); + const { kind } = await report("environment_teardown_claim"); + expect(kind).toMatchObject({ verdicts: { gone: 1 }, reasons: { pid_dead: 1 } }); + }); + + it("never reads a controller stamped for an earlier attempt: the claim is a fence held by its token", async () => { + const attemptId = randomUUID(); + await claim({ pendingCleanupAttemptId: attemptId, pendingCleanupLeaseExpiresAtMs: Date.now() + 60_000, + pendingCleanupController: { ...currentControllerProcessStamp(), bootId: "earlier-boot", attemptId: randomUUID() } }); + const { kind, violations } = await report("environment_teardown_claim"); + expect(kind).toMatchObject({ verdicts: { alive: 1 }, reasons: { fence_held: 1 } }); + expect(violations).toEqual([]); + }); + + it("keeps an unstamped claim whose fence expired unprovable, never gone", async () => { + await claim({ pendingCleanupAttemptId: randomUUID(), pendingCleanupLeaseExpiresAtMs: Date.now() - 1 }); + const { kind } = await report("environment_teardown_claim"); + expect(kind).toMatchObject({ verdicts: { unprovable: 1 }, reasons: { fence_expired_without_provider_proof: 1 } }); + }); + + it("names no owner for an in-flight flag without an attempt", async () => { + await claim({}); + const { kind, violations } = await report("environment_teardown_claim"); + expect(kind).toMatchObject({ verdicts: { unprovable: 1 }, reasons: { no_owner_stamp: 1 } }); + expect(violations.map(violation => violation.rule)).toEqual(["I1_owned"]); + }); + }); +}); diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index e0fbe89ce7d..c4fc7da85b0 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -319,6 +319,34 @@ function runLeaseKind(): OwnableKind { }; } +/** + * Owner of an in-flight teardown claim. A claimant that stamps + * `pendingCleanupController` for this exact attempt is a process, proved by + * identity. Other claimants (workspace recovery, native export retry, and + * claims saved by an earlier release) hold the claim as a fence: the attempt + * id is the token and `pendingCleanupLeaseExpiresAtMs` is the deadline that + * the claimant renews. An expired fence is never proof by itself; another + * host takes over only through the fence. A controller stamp left by an + * earlier attempt names a different claimant and is ignored. + */ +function teardownClaimOwner(row: LeaseRow): RecordOwner { + const metadata = json(row.metadata); + const attemptId = str(metadata.pendingCleanupAttemptId); + const controller = json(metadata.pendingCleanupController); + if (attemptId && controller.attemptId === attemptId) { + const stamped = controllerStamp(controller, "environment_teardown_claim"); + if (stamped.kind !== "none") return stamped; + } + if (!attemptId) return { kind: "none" }; + const expiresAtMs = typeof metadata.pendingCleanupLeaseExpiresAtMs === "number" ? metadata.pendingCleanupLeaseExpiresAtMs : null; + return { + kind: "fence", + stamp: { kind: "remote", scope: "environment_teardown_claim", generation: null, fenceToken: attemptId }, + fence: { currentFenceToken: attemptId, currentGeneration: null, + leaseExpiresAt: expiresAtMs !== null && Number.isFinite(expiresAtMs) ? new Date(expiresAtMs) : null }, + }; +} + /** `ControllerProcess` metadata ({bootId, pid, hostHash, processStartedAt}). */ function controllerStamp(value: unknown, scope: string): RecordOwner { const controller = readControllerProcess(value); @@ -554,7 +582,7 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ runLeaseKind(), leaseKind("environment_teardown_claim", "Teardown claim (in-flight cleanup)", "ownership", and(eq(environmentLeases.status, "pending_cleanup"), sql`(${environmentLeases.metadata}->>'pendingCleanupInFlight')::boolean is true`), - row => controllerStamp(json(row.metadata).pendingCleanupController, "environment_teardown_claim")), + teardownClaimOwner), leaseKind("environment_pending_cleanup", "Pending-cleanup obligation (not in flight)", "ownership", and(eq(environmentLeases.status, "pending_cleanup"), sql`coalesce((${environmentLeases.metadata}->>'pendingCleanupInFlight')::boolean, false) is false`), @@ -569,7 +597,10 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ eq(environmentLeases.status, "retained"), () => ({ kind: "none" }), [1, 7]), spoolKind("sandbox_orphan_cleanup_spool", "Orphan cleanup buffer/spool", "ownership", idleOrphanSpoolPath), leaseKind("environment_login_probe_lease", "Login/probe lease", "ownership", - and(eq(environmentLeases.status, "active"), isNull(environmentLeases.heartbeatRunId)), () => ({ kind: "none" })), + and(eq(environmentLeases.status, "active"), isNull(environmentLeases.heartbeatRunId)), + // Stamped by the write that makes the lease active. A lease saved before + // the stamp existed names no owner; it ends with its login session. + row => controllerStamp(json(row.metadata)[ENVIRONMENT_LEASE_CONTROLLER_KEY], "environment_login_probe_lease")), // Workspaces operationKind("workspace_runtime_control_operation", "Managed runtime-control operation", "ownership",