diff --git a/server/src/__tests__/ownership-reconciliation-report.test.ts b/server/src/__tests__/ownership-reconciliation-report.test.ts index c05f83521ee..f60e77538c7 100644 --- a/server/src/__tests__/ownership-reconciliation-report.test.ts +++ b/server/src/__tests__/ownership-reconciliation-report.test.ts @@ -143,7 +143,7 @@ describe("report-only ownership reconciliation", () => { expect(new Set(names).size).toBe(names.length); expect(names.length).toBeGreaterThanOrEqual(40); expect(OWNABLE_KINDS.filter(kind => kind.settle !== null).map(kind => kind.id)) - .toEqual(["native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "sandbox_creation_intent", "environment_run_lease_active", "environment_teardown_claim", "environment_retained_orphan_sandbox", "workspace_runtime_control_operation", "native_workspace_finalize_attempt", "execution_workspace_runtime_lease", "copyback_owner_hold", "observed_workspace_preservation_fence", "wake_request_claimed", "legacy_controller_lease", "issue_execution_lock", "issue_checkout_lock"]); + .toEqual(["native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "sandbox_creation_intent", "environment_run_lease_active", "environment_teardown_claim", "environment_retained_orphan_sandbox", "workspace_runtime_control_operation", "native_workspace_finalize_attempt", "execution_workspace_runtime_lease", "legacy_shutdown_settlement_fence", "copyback_owner_hold", "observed_workspace_preservation_fence", "wake_request_claimed", "legacy_controller_lease", "issue_execution_lock", "issue_checkout_lock"]); expect(OWNABLE_KINDS.filter(kind => !kind.listInProgress).every(kind => kind.coverageGap)).toBe(true); }); diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index d8586a3b6b1..dc27c7641a5 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -18,6 +18,9 @@ import { claimNativeRestartRecoveries, currentNativeControllerIdentity, NATIVE_R import { startNativeSessionExecutionLeaseRenewal } from "../native-runtime/native-execution-lease.js"; import { allowLegacyShutdownWorkspaceCleanup, beginLegacyShutdownWorkspaceSettlement, finishLegacyShutdownWorkspaceSettlement, maintainLegacyShutdownWorkspaceSettlement, legacyShutdownWorkspaceResourceProtected, sweepLegacyShutdownWorkspaceSettlements } from "../legacy-shutdown-workspace-settlement.js"; import { adapterExecutionControls, createAdapterExecutionControl } from "../adapter-execution-control.js"; +import { currentControllerProcessStamp, currentProcessSpaceHash, currentProcessStartedAtMs } from "../controller-process-identity.js"; +import { createOwnershipReportState, runOwnershipReconciliationPass } from "../recovery/ownership-invariant-report.js"; +import { OWNABLE_KINDS } from "../recovery/ownership-registry.js"; import { logger } from "../../middleware/logger.js"; import { listExpiredNativeRunnerAuthorities } from "../native-runtime/native-runner-checkpoint.js"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../delivery-work-notifications.js"; @@ -847,6 +850,69 @@ describePostgres("heartbeat recovery module ownership", () => { adapterExecutionControls.delete(f.run.id); } + async function fenceFromPreviousServer(f: Awaited>, + controller: Record | null) { + const [row] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + const marker: Record = { ...(row.metadata!.legacyShutdownWorkspaceSettlement as Record), + controllerBootId: randomUUID() }; + if (controller) marker.controller = { ...controller, bootId: marker.controllerBootId }; + else delete marker.controller; + await db.update(environmentLeases).set({ metadata: { ...row.metadata, legacyShutdownWorkspaceSettlement: marker } }) + .where(eq(environmentLeases.id, f.lease.id)); + await db.update(heartbeatRuns).set({ status: "interrupted", errorCode: "server_shutdown_interrupted", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, f.run.id)); + } + const fenceState = async (leaseId: string) => ((await db.select().from(environmentLeases) + .where(eq(environmentLeases.id, leaseId)))[0].metadata!.legacyShutdownWorkspaceSettlement as { state: string }); + const deadController = () => ({ pid: 4_194_303, hostHash: currentProcessSpaceHash(), processStartedAt: new Date(0).toISOString() }); + + it("stamps the shutdown fence with the committing server process", async () => { + await currentProcessStartedAtMs(); + const f = await seedShutdownSource(); + try { + const controller = currentControllerProcessStamp(); + expect((await fenceState(f.lease.id) as unknown as { controller: unknown }).controller) + .toEqual({ bootId: controller.bootId, pid: process.pid, hostHash: controller.hostHash, + processStartedAt: controller.processStartedAt }); + } finally { await deleteShutdownSource(f); } + }); + + it("settles the fence of a previous server proven gone without waiting out its deadline", async () => { + const f = await seedShutdownSource(); + try { + await fenceFromPreviousServer(f, deadController()); + await sweepLegacyShutdownWorkspaceSettlements(db); + expect(await fenceState(f.lease.id)).toMatchObject({ state: "expired" }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].resultJson) + .toMatchObject({ workspaceRestoreFailure: "restore_failed" }); + } finally { await deleteShutdownSource(f); } + }); + + it("settles a proven-gone fence owner from the shared reconciler, and only that one", async () => { + const dead = await seedShutdownSource(); + const foreign = await seedShutdownSource(); + const legacy = await seedShutdownSource(); + const kind = OWNABLE_KINDS.find(entry => entry.id === "legacy_shutdown_settlement_fence")!; + try { + await fenceFromPreviousServer(dead, deadController()); + await fenceFromPreviousServer(foreign, { ...deadController(), hostHash: "another-host-pid-space" }); + await fenceFromPreviousServer(legacy, null); + const readOnly = await runOwnershipReconciliationPass(db, { state: createOwnershipReportState(), kinds: [kind], record: false }); + expect(readOnly.kinds[0]).toMatchObject({ inProgress: 3, settled: 0 }); + expect(await fenceState(dead.lease.id)).toMatchObject({ state: "pending" }); + + const report = await runOwnershipReconciliationPass(db, { state: createOwnershipReportState(), kinds: [kind] }); + expect(report.kinds[0]).toMatchObject({ inProgress: 3, settled: 1 }); + expect(await fenceState(dead.lease.id)).toMatchObject({ state: "expired" }); + await sweepLegacyShutdownWorkspaceSettlements(db); + expect(await fenceState(foreign.lease.id)).toMatchObject({ state: "pending" }); + expect(await fenceState(legacy.lease.id)).toMatchObject({ state: "pending" }); + await sweepLegacyShutdownWorkspaceSettlements(db, new Date(Date.now() + 60_000)); + expect(await fenceState(foreign.lease.id)).toMatchObject({ state: "expired" }); + expect(await fenceState(legacy.lease.id)).toMatchObject({ state: "expired" }); + } finally { for (const f of [dead, foreign, legacy]) await deleteShutdownSource(f); } + }); + it("does not let an elapsed shutdown wait stop a sandbox whose exporter still owns settlement", async () => { const f = await seedShutdownSource(); const control = createAdapterExecutionControl(); diff --git a/server/src/services/legacy-shutdown-workspace-settlement.ts b/server/src/services/legacy-shutdown-workspace-settlement.ts index c4a4451c364..7f182149929 100644 --- a/server/src/services/legacy-shutdown-workspace-settlement.ts +++ b/server/src/services/legacy-shutdown-workspace-settlement.ts @@ -5,6 +5,8 @@ import { adapterExecutionControls } from "./adapter-execution-control.js"; import { legacyControllerBootId } from "./legacy-controller-lease.js"; import { LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; import { logger } from "../middleware/logger.js"; +import { currentControllerProcessStamp, readControllerProcess } from "./controller-process-identity.js"; +import { proveOwnerGone, type LocalOwnerStamp, type OwnerProof } from "./owner-proof.js"; const SCHEMA = "paperclip.legacy-shutdown-workspace-settlement.v1"; type Lease = Pick; @@ -61,6 +63,30 @@ export function pendingLegacyShutdownWorkspaceSettlement(lease: Lease) { && typeof marker.deadlineAt === "string" ? marker : null; } +export function legacyShutdownSettlementOwnerStamp(metadata: unknown): LocalOwnerStamp | null { + if (!metadata || typeof metadata !== "object") return null; + const marker = (metadata as Record).legacyShutdownWorkspaceSettlement; + if (!marker || typeof marker !== "object" || Array.isArray(marker)) return null; + const controller = readControllerProcess((marker as Record).controller); + if (!controller) return null; + return { kind: "local", scope: "legacy_shutdown_settlement_fence", generation: null, host: null, + processSpace: controller.hostHash, bootId: controller.bootId, pid: controller.pid, + processStartedAt: controller.processStartedAt, processGroupId: null }; +} + +export function legacyShutdownSettlementOwner(metadata: unknown): + { kind: "process"; stamp: LocalOwnerStamp } | { kind: "none" } { + const stamp = legacyShutdownSettlementOwnerStamp(metadata); + if (stamp) return { kind: "process", stamp }; + const marker = (metadata as { legacyShutdownWorkspaceSettlement?: { controllerBootId?: unknown } } | null)?.legacyShutdownWorkspaceSettlement; + const bootId = typeof marker?.controllerBootId === "string" && marker.controllerBootId ? marker.controllerBootId : null; + return bootId ? { kind: "process", stamp: { kind: "local", scope: "legacy_shutdown_settlement_fence", generation: null, + host: null, processSpace: null, bootId, pid: null, processStartedAt: null } } : { kind: "none" }; +} + +const sameOwner = (left: LocalOwnerStamp, right: LocalOwnerStamp) => left.bootId === right.bootId + && left.pid === right.pid && left.processSpace === right.processSpace && left.processStartedAt === right.processStartedAt; + /** Historical reuse rows may name the same allocation as the fenced source. */ export async function legacyShutdownWorkspaceResourceProtected(db: Db, lease: Pick) { @@ -84,6 +110,34 @@ export async function sweepLegacyShutdownWorkspaceSettlements(db: Db, now = new sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt' <= ${now.toISOString()}`, )).orderBy(sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt'`).limit(50); for (const lease of rows) await allowLegacyShutdownWorkspaceCleanup(db, lease, now, { onDeferredDeadline: scope.onDeferredDeadline }); + const held = await db.select().from(environmentLeases).where(and( + scope.companyId ? eq(environmentLeases.companyId, scope.companyId) : undefined, + scope.leaseId ? eq(environmentLeases.id, scope.leaseId) : undefined, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt' > ${now.toISOString()}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'controllerBootId' <> ${legacyControllerBootId}`, + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement' ? 'controller'`, + )).limit(50); + for (const lease of held) { + const owner = legacyShutdownSettlementOwnerStamp(lease.metadata); + if (!owner || (await proveOwnerGone(owner)).verdict !== "gone") continue; + await allowLegacyShutdownWorkspaceCleanup(db, lease, now, { onDeferredDeadline: scope.onDeferredDeadline, + ownerProvenGone: owner }); + } +} + +export async function settleLegacyShutdownSettlementFence(db: Db, record: { recordId: string; owner: { kind: string; stamp?: unknown } }, + context: { now: Date; proof: OwnerProof }): Promise<{ settled: boolean; reason: string }> { + if (record.owner.kind !== "process") return { settled: false, reason: "owner_not_provable" }; + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, record.recordId)); + if (!lease || !pendingLegacyShutdownWorkspaceSettlement(lease)) return { settled: false, reason: "fence_no_longer_pending" }; + const owner = legacyShutdownSettlementOwnerStamp(lease.metadata); + if (!owner || !sameOwner(owner, record.owner.stamp as LocalOwnerStamp)) return { settled: false, reason: "owner_changed_since_proof" }; + if (!await allowLegacyShutdownWorkspaceCleanup(db, lease, context.now, { ownerProvenGone: owner })) { + return { settled: false, reason: "source_not_settleable_yet" }; + } + return { settled: true, reason: context.proof.reason }; } /** Commit the fence while the run is still live, before exposing interruption. */ @@ -92,7 +146,8 @@ export async function beginLegacyShutdownWorkspaceSettlement(db: Db, run: Run, d await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: run.id }); await tx.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ legacyShutdownWorkspaceSettlement: { schema: SCHEMA, state: "pending", runId: run.id, - controllerBootId: legacyControllerBootId, deadlineAt: deadlineAt.toISOString() } })}::jsonb`, + controllerBootId: legacyControllerBootId, controller: currentControllerProcessStamp(), + deadlineAt: deadlineAt.toISOString() } })}::jsonb`, }).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.status, "active"), ne(environmentLeases.provider, "local"), or(sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`, sql`${environmentLeases.metadata}->>'sandboxProviderPlugin' = 'true'`))); @@ -115,7 +170,10 @@ export async function finishLegacyShutdownWorkspaceSettlement(db: Db, run: Run, /** A restart or another cleanup worker must never infer saved files from timeout. */ export async function allowLegacyShutdownWorkspaceCleanup(db: Db, lease: Lease, now = new Date(), - options: { ownerStopOnly?: boolean; onDeferredDeadline?: (at: number) => void } = {}) { + options: { + ownerStopOnly?: boolean; onDeferredDeadline?: (at: number) => void; + ownerProvenGone?: LocalOwnerStamp; + } = {}) { if (!pendingLegacyShutdownWorkspaceSettlement(lease)) return true; return db.transaction(async tx => { const [hint] = await tx.select().from(environmentLeases).where(and( @@ -144,8 +202,11 @@ export async function allowLegacyShutdownWorkspaceCleanup(db: Db, lease: Lease, if (options.ownerStopOnly && marker.controllerBootId === legacyControllerBootId && adapterExecutionControls.get(run.id)?.controller.signal.aborted) return true; if (marker.controllerBootId === legacyControllerBootId && adapterExecutionControls.has(run.id)) return false; + const markerOwner = legacyShutdownSettlementOwnerStamp(current.metadata); + const ownerGone = !!options.ownerProvenGone && marker.controllerBootId !== legacyControllerBootId + && !!markerOwner && sameOwner(markerOwner, options.ownerProvenGone); const deadline = Date.parse(marker.deadlineAt as string); - if (!Number.isFinite(deadline) || deadline > now.getTime()) return false; + if (!ownerGone && (!Number.isFinite(deadline) || deadline > now.getTime())) return false; // A killed container cannot certify export. Preserve source and hold in // this same transaction before any worker may perform provider cleanup. if (current.status !== "active") { diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index bd9dff77519..f1ab189929f 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -36,6 +36,8 @@ import { settleAbandonedWorkspaceFinalizeAttempt } from "./workspace-finalize-at import { runtimeControlOperationOwnerStamp, settleRuntimeControlOperation } from "../workspace-operations.js"; import { copybackOwnerHoldOwner, settleCopybackOwnerHold } from "./copyback-owner-hold.js"; import { applyObservedFenceProviderProof, observedWorkspaceFenceOwner, settleObservedWorkspaceFence } from "./observed-workspace-fence.js"; +import { copybackOwnerStamp } from "../native-runtime/native-workspace-owner-stop-proof.js"; +import { legacyShutdownSettlementOwner, settleLegacyShutdownSettlementFence } from "../legacy-shutdown-workspace-settlement.js"; import { nativeControllerOwnerStamp, runProcessOwnerStamp } from "../native-runtime/native-owner-stamps.js"; import { settleIssueLock } from "./issue-lock-settlement.js"; import { legacyControllerOwnerStamp, settleLegacyControllerLease } from "../legacy-controller-lease.js"; @@ -838,13 +840,12 @@ export const OWNABLE_KINDS: readonly OwnableKind[] = [ })) }; }, }, - leaseKind("legacy_shutdown_settlement_fence", "Legacy graceful-shutdown workspace settlement fence", "workspace_preservation", - sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, - row => { - const bootId = str(json(json(row.metadata).legacyShutdownWorkspaceSettlement).controllerBootId); - return bootId ? { kind: "process", stamp: { kind: "local", scope: "legacy_shutdown_settlement_fence", - generation: null, host: null, processSpace: null, bootId, pid: null, processStartedAt: null } } : { kind: "none" }; - }, [2]), + { + ...leaseKind("legacy_shutdown_settlement_fence", "Legacy graceful-shutdown workspace settlement fence", "workspace_preservation", + sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`, + row => legacyShutdownSettlementOwner(row.metadata), [2]), + settle: settleLegacyShutdownSettlementFence, + }, leaseKind("legacy_restore_recovery_hold", "Legacy restore-recovery preservation hold", "workspace_preservation", and(eq(environmentLeases.leasePolicy, "retain_on_failure"), sql`${environmentLeases.metadata} ? 'workspaceRestoreRecovery'`, notInArray(environmentLeases.status, ["released", "expired", "failed"])),