diff --git a/packages/db/src/client-options.test.ts b/packages/db/src/client-options.test.ts index a6fbce00588..3e4836e7ad8 100644 --- a/packages/db/src/client-options.test.ts +++ b/packages/db/src/client-options.test.ts @@ -93,6 +93,12 @@ describe("databaseClientOptionsFromEnv", () => { connection: { application_name: "paperclip-web" }, }); }); + + it("sends a dedicated pool's statement timeout as a startup parameter", () => { + expect(postgresJsOptions({ applicationName: "paperclip-controller-instance-lease", statementTimeoutMs: 8_000 })).toEqual({ + connection: { application_name: "paperclip-controller-instance-lease", statement_timeout: "8000" }, + }); + }); }); describe("resolveDatabaseClientOptions", () => { diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index aa4f0c9fd06..0a38039de4b 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -163,6 +163,11 @@ export interface DatabaseClientOptions { * `postgres.js`). */ applicationName?: string; + /** + * Connection-level `statement_timeout` in milliseconds, sent as a startup + * parameter (no extra round trip). For dedicated single-purpose pools only. + */ + statementTimeoutMs?: number; } /** @@ -256,9 +261,10 @@ export function postgresJsOptions(options: DatabaseClientOptions): Record = {}; + if (options.applicationName !== undefined) connection.application_name = options.applicationName; + if (options.statementTimeoutMs !== undefined) connection.statement_timeout = String(options.statementTimeoutMs); + if (Object.keys(connection).length) driverOptions.connection = connection; return driverOptions; } @@ -324,11 +330,42 @@ export function controllerLeaseDb(db: Db): Db { return created; } +const controllerInstanceLeaseDbFactories = new WeakMap Db>(); +const controllerInstanceLeaseDbs = new WeakMap(); + +/** One private connection for the server's controller-instance lease, so its + * renewal never waits behind request or worker traffic on the main pool. + * `statementTimeoutMs` (first call wins) is a connection-level + * `statement_timeout`: the server cancels a slow renewal, so it never holds the + * single connection that the next renewal needs, at no extra round trip. */ +export function controllerInstanceLeaseDb(db: Db, options: { statementTimeoutMs?: number | null } = {}): Db { + const existing = controllerInstanceLeaseDbs.get(db); + if (existing) return existing; + const factory = controllerInstanceLeaseDbFactories.get(db); + if (!factory) return db; + const created = factory(options.statementTimeoutMs ?? null); + controllerInstanceLeaseDbs.set(db, created); + return created; +} + +/** Replaces the lease connection with one that sends no `statement_timeout` + * startup parameter, for a connection pooler that refuses it. */ +export function controllerInstanceLeaseDbWithoutStatementTimeout(db: Db): Db { + const factory = controllerInstanceLeaseDbFactories.get(db); + if (!factory) return db; + const previous = controllerInstanceLeaseDbs.get(db); + const created = factory(null); + controllerInstanceLeaseDbs.set(db, created); + void previous?.$client.end({ timeout: 0 }).catch(() => undefined); + return created; +} + /** Ends the liveness pool if one was created. The ended pool stays cached, so * a late renewal fails like a query on the ended primary pool. */ export async function closeLivenessDb(db: Db, timeoutSeconds: number): Promise { await Promise.all([livenessDbs.get(db)?.$client.end({ timeout: timeoutSeconds }), - controllerLeaseDbs.get(db)?.$client.end({ timeout: timeoutSeconds })]); + controllerLeaseDbs.get(db)?.$client.end({ timeout: timeoutSeconds }), + controllerInstanceLeaseDbs.get(db)?.$client.end({ timeout: timeoutSeconds })]); } // Control checkpoints must not occupy the request or lease-renewal pools @@ -406,6 +443,10 @@ function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: ob controllerLeaseDbFactories.set(db, () => createDbClient(url, { ...resolved, maxConnections: 1, applicationName: "paperclip-legacy-controller-lease", }, db)); + controllerInstanceLeaseDbFactories.set(db, statementTimeoutMs => createDbClient(url, { + ...resolved, maxConnections: 1, applicationName: "paperclip-controller-instance-lease", + ...(statementTimeoutMs !== null ? { statementTimeoutMs } : {}), + }, db)); checkpointDbFactories.set(db, () => createDbClient(url, { ...resolved, maxConnections: CHECKPOINT_POOL_MAX_CONNECTIONS, applicationName: "paperclip-runner-checkpoint", }, db)); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index feb4848b43e..0c4c0fcd9e8 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -5,6 +5,8 @@ export { type OwnedDatabaseListener, livenessDb, controllerLeaseDb, + controllerInstanceLeaseDb, + controllerInstanceLeaseDbWithoutStatementTimeout, closeLivenessDb, checkpointDb, closeCheckpointDb, diff --git a/server/src/__tests__/controller-instance-lease.test.ts b/server/src/__tests__/controller-instance-lease.test.ts index 9552e66e666..069ab8c679b 100644 --- a/server/src/__tests__/controller-instance-lease.test.ts +++ b/server/src/__tests__/controller-instance-lease.test.ts @@ -1,10 +1,10 @@ import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, companies, controllerInstances, createDb, environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db"; +import { agents, companies, controllerInstanceLeaseDb, controllerInstances, createDb, environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, workspaceOperations } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { - controllerInstanceLeaseEnabled, proveControllerInstance, registerControllerInstanceDb, registerControllerInstanceLeaseWork, + RENEW_STATEMENT_TIMEOUT_MS, controllerInstanceLeaseEnabled, proveControllerInstance, registerControllerInstanceDb, registerControllerInstanceLeaseWork, resetControllerInstanceLeaseForTests, stopControllerInstanceLease, tickControllerInstanceLease, } from "../services/controller-instance-lease.js"; import { currentControllerBootId, currentControllerProcessStamp, currentProcessSpaceHash } from "../services/controller-process-identity.js"; @@ -224,6 +224,91 @@ suite("controller instance lease: a replaced container's records are provable", } finally { vi.useRealTimers(); } }); + it("renews on its own connection while the main pool is saturated (deploy overlap traffic switch)", async () => { + // One main-pool connection, held by a slow statement: before the fix the renewal + // waited behind it, timed out, and five such timeouts made a healthy server exit. + const busy = createDb(temporary!.connectionString, { maxConnections: 1 }); + try { + registerControllerInstanceDb(busy); + const lost = vi.fn(); + registerControllerInstanceLeaseWork(busy, coordinator, { onLost: lost }); + currentControllerProcessStamp(); + await tickControllerInstanceLease(); + // Drizzle queries are lazy: attach a continuation so the statement starts now. + const hold = busy.execute(sql`select pg_sleep(12)`).then(() => undefined); + await new Promise(resolve => setTimeout(resolve, 200)); + const [before] = await db.select({ renewedAt: controllerInstances.renewedAt }).from(controllerInstances) + .where(eq(controllerInstances.bootId, currentControllerBootId())); + currentControllerProcessStamp(); + const started = Date.now(); + await tickControllerInstanceLease(); + expect(Date.now() - started).toBeLessThan(5_000); + const [after] = await db.select({ renewedAt: controllerInstances.renewedAt }).from(controllerInstances) + .where(eq(controllerInstances.bootId, currentControllerBootId())); + expect(after.renewedAt!.getTime()).toBeGreaterThan(before.renewedAt!.getTime()); + expect(lost).not.toHaveBeenCalled(); + await hold; + } finally { + await stopControllerInstanceLease(busy); + await busy.$client.end({ timeout: 5 }); + } + }, 30_000); + + it("has the server cancel a renewal blocked on a row lock, and renews on the same connection next time", async () => { + // Five renewals failed 15 s apart: a client-timed-out statement kept running on the + // single lease connection and every next renewal queued behind it. + const lockDb = createDb(temporary!.connectionString, { maxConnections: 1 }); + // Resolving twice is harmless, so the normal path and the cleanup both release. + let release: () => void = () => {}; + const released = new Promise(resolve => { release = resolve; }); + let holder: Promise = Promise.resolve(); + try { + registerControllerInstanceDb(db); + registerControllerInstanceLeaseWork(db, coordinator, { onLost: vi.fn() }); + currentControllerProcessStamp(); + await tickControllerInstanceLease(); + const lease = controllerInstanceLeaseDb(db); + const backend = async () => (await lease.execute(sql`select pg_backend_pid() as pid`) as unknown as Array<{ pid: number }>)[0]!.pid; + const pidBefore = await backend(); + const [before] = await db.select({ renewedAt: controllerInstances.renewedAt }).from(controllerInstances) + .where(eq(controllerInstances.bootId, currentControllerBootId())); + let locked!: () => void; + const lockTaken = new Promise(resolve => { locked = resolve; }); + holder = lockDb.transaction(async tx => { + await tx.execute(sql`select 1 from controller_instances where boot_id = ${currentControllerBootId()} for update`); + locked(); + await released; + }); + await lockTaken; + currentControllerProcessStamp(); + const started = Date.now(); + await tickControllerInstanceLease(); + const elapsed = Date.now() - started; + expect(elapsed).toBeGreaterThanOrEqual(RENEW_STATEMENT_TIMEOUT_MS - 500); + expect(elapsed).toBeLessThan(RENEW_STATEMENT_TIMEOUT_MS + 1_500); + // Cancelled by the server, not left running behind the client timeout. + const running = await db.execute(sql`select count(*)::int as count from pg_stat_activity + where application_name = 'paperclip-controller-instance-lease' and state = 'active'`) as unknown as Array<{ count: number }>; + expect(running[0]!.count).toBe(0); + release(); + await holder; + currentControllerProcessStamp(); + const next = Date.now(); + await tickControllerInstanceLease(); + expect(Date.now() - next).toBeLessThan(2_000); + const [after] = await db.select({ renewedAt: controllerInstances.renewedAt }).from(controllerInstances) + .where(eq(controllerInstances.bootId, currentControllerBootId())); + expect(after.renewedAt!.getTime()).toBeGreaterThan(before.renewedAt!.getTime()); + expect(await backend()).toBe(pidBefore); + } finally { + // Free the controller row before the stop record writes it. + release(); + await holder.catch(() => undefined); + await stopControllerInstanceLease(db); + await lockDb.$client.end({ timeout: 5 }); + } + }, 40_000); + it("proves a cleanly stopped boot gone at once", async () => { const stopped = await otherBoot({ liveForMs: 60_000 }); await db.update(controllerInstances).set({ stoppedAt: sql`now()` }).where(eq(controllerInstances.bootId, stopped)); diff --git a/server/src/__tests__/heartbeat-native-restart-suspension.test.ts b/server/src/__tests__/heartbeat-native-restart-suspension.test.ts index ce5753dea9e..18a324f2963 100644 --- a/server/src/__tests__/heartbeat-native-restart-suspension.test.ts +++ b/server/src/__tests__/heartbeat-native-restart-suspension.test.ts @@ -37,7 +37,7 @@ import { currentControllerBootId, currentProcessSpaceHash } from "../services/co import { createOwnershipReportState, runOwnershipReconciliationPass, settleOwnableKind, type OwnershipReport } from "../services/recovery/ownership-invariant-report.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { reconcileNativeFinalizations, registerOwnershipReconciliationWork } from "../services/native-runtime/native-finalization-reconciler.js"; -import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; +import { OWNABLE_KINDS, registerNativeRestartRecheck } from "../services/recovery/ownership-registry.js"; import { currentProcessProbe, proveOwnerGone } from "../services/owner-proof.js"; import { createRunLogWriterAuthority } from "../services/heartbeat/run-log-authority.js"; import { resolveHotRestartIntentPath, writeHotRestartIntent } from "../services/hot-restart.js"; @@ -948,6 +948,82 @@ describeEmbeddedPostgres("heartbeat native recovery interrupted by graceful shut } finally { probe.restore(); unknown?.mockRestore(); } }, 60_000); + it("ends an unverifiable run on the first look once its controller's lease lapsed longer ago than the bound", async () => { + // A successor that boots (or first scans) minutes after the old server left + // must not restart the 2-minute wait from its own first observation. + const f = await seedClaim("CLAIM-SPACE-LAPSED-LONG-AGO", "expired", "foreign-process-space"); + await f.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 3 * 60_000) }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const probe = observeOwnerPid(), onClaim = vi.fn(); + try { + const dispositions = await claimNativeRestartRecoveries({ db: f.db, restartKind: "hard", runIds: [f.runId], onClaim }); + expect(dispositions).toEqual([{ kind: "blocked", runId: f.runId, reason: "controller_process_space_unverified" }]); + expect(probe.observed).toEqual([]); + expect(onClaim).not.toHaveBeenCalled(); + expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0]) + .toMatchObject({ status: "failed", errorCode: "native_restart_recovery_blocked" }); + } finally { probe.restore(); } + }, 60_000); + + it("ends at once a run whose controller boot is fenced, even with its 20-minute claim lease unexpired (N1)", async () => { + // N1 cell: old server SIGKILLed mid-overlap at 22:09:31; its boot was fenced at 22:13:02, and the + // run still waited a further 2 minutes from that first look. A fenced boot already outlived its lease. + const f = await seedClaim("CLAIM-SPACE-FENCED-BOOT", "live", "foreign-process-space"); + const [owner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + await f.db.insert(controllerInstances).values({ bootId: owner!.controllerBootId!, processSpace: "foreign-process-space", + pid: priorPid, leaseExpiresAt: new Date(Date.now() - 3 * 60_000), fencedAt: new Date(Date.now() - 10_000) }); + registerControllerInstanceDb(f.db); + const probe = observeOwnerPid(); + try { + expect(await claimNativeRestartRecoveries({ db: f.db, restartKind: "hard", runIds: [f.runId], onClaim: vi.fn() })) + .toEqual([{ kind: "blocked", runId: f.runId, reason: "controller_process_space_unverified" }]); + } finally { probe.restore(); resetControllerInstanceLeaseForTests(); } + }, 60_000); + + it("keeps waiting for a controller whose boot is still live, whatever the run's own lease says", async () => { + const f = await seedClaim("CLAIM-SPACE-LIVE-BOOT", "expired", "foreign-process-space"); + const [owner] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + await f.db.insert(controllerInstances).values({ bootId: owner!.controllerBootId!, processSpace: "foreign-process-space", + pid: priorPid, leaseExpiresAt: new Date(Date.now() + 60_000) }); + registerControllerInstanceDb(f.db); + const probe = observeOwnerPid(); + try { + expect(await claimNativeRestartRecoveries({ db: f.db, restartKind: "hard", runIds: [f.runId], onClaim: vi.fn() })) + .toMatchObject([{ kind: "awaiting_evidence", runId: f.runId }]); + expect((await f.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)))[0].status).toBe("running"); + } finally { probe.restore(); resetControllerInstanceLeaseForTests(); } + }, 60_000); + + it("never moves the owner's last sign of life when a successor records its wait (fence time unchanged)", async () => { + // N1: the successor's awaiting-evidence write bumped updated_at, which the prover reads as the dead + // owner's last write, and the fence slipped from about 2 minutes to 3.5 after the crash. + const f = await seedClaim("CLAIM-SPACE-OWNER-TOUCH", "expired", "foreign-process-space"); + const ownerLastWrite = new Date(Date.now() - 5 * 60_000); + await f.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 60_000), updatedAt: ownerLastWrite }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + const probe = observeOwnerPid(); + try { + expect(await claimNativeRestartRecoveries({ db: f.db, restartKind: "hard", runIds: [f.runId], onClaim: vi.fn() })) + .toMatchObject([{ kind: "awaiting_evidence", reason: "controller_process_space_unverified" }]); + const [after] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(after!.recoveryState).toBe("awaiting_evidence"); + expect(after!.updatedAt.getTime()).toBe(ownerLastWrite.getTime()); + } finally { probe.restore(); } + }, 60_000); + + it("schedules the recheck for the end of the bound measured from the lease lapse", async () => { + const f = await seedClaim("CLAIM-SPACE-LAPSED-RECENTLY", "expired", "foreign-process-space"); + const lapsedAt = Date.now() - 30_000; + await f.db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(lapsedAt) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const probe = observeOwnerPid(); + try { + const [disposition] = await claimNativeRestartRecoveries({ db: f.db, restartKind: "hard", runIds: [f.runId], onClaim: vi.fn() }); + expect(disposition).toMatchObject({ kind: "awaiting_evidence", reason: "controller_process_space_unverified" }); + const retryAt = (disposition as { retryAt: Date }).retryAt.getTime(); + expect(Math.abs(retryAt - (lapsedAt + 2 * 60_000))).toBeLessThan(5_000); + } finally { probe.restore(); } + }, 60_000); + it("keeps a real controller-only attestation from authorizing an in-flight claim", async () => { const f = await seedClaim("CLAIM-CONTROLLER-ONLY", "expired", "foreign-process-space"); const [before] = await f.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); @@ -1686,6 +1762,35 @@ describeEmbeddedPostgres("heartbeat native recovery interrupted by graceful shut expect(second.violations.filter(violation => violation.kind === "native_accepted_result_lease")).toEqual([]); }, 60_000); + it("hands a run awaiting evidence to restart recovery once the reconciler proves its controller gone (crash mid-overlap)", async () => { + // A crashed old server's run was classified by the new server's startup pass while the + // old server lived, so no restart re-check was scheduled for it. Only the reconciler + // sees the controller die; it must hand the run to restart recovery on the live server. + const f = await seedRecoverableRun("AWAITING-CRASHED-CONTROLLER"); + await f.db.update(nativeRunFinalizations).set({ recoveryState: "awaiting_evidence", + leaseOwner: `old-controller:${randomUUID()}`, leaseExpiresAt: new Date(Date.now() + 15 * 60_000), + controllerBootId: randomUUID(), controllerPid: 2_000_000_001, controllerHostHash: currentProcessSpaceHash(), + controllerProcessStartedAt: new Date(Date.now() - 60_000) }).where(eq(nativeRunFinalizations.runId, f.runId)); + const kind = OWNABLE_KINDS.find(candidate => candidate.id === "native_awaiting_evidence")!; + const kinds = [{ ...kind, listInProgress: async (db: typeof f.db) => ({ records: await kind.listForRuns!(db, [f.runId]), truncated: false }) }]; + const recheck = vi.fn(async () => undefined); + const unregister = registerNativeRestartRecheck(recheck); + const original = process.kill.bind(process); + const kill = vi.spyOn(process, "kill").mockImplementation((pid, signal) => { + if (pid === 2_000_000_001 && signal === 0) throw Object.assign(new Error("gone"), { code: "ESRCH" }); + return original(pid, signal); + }); + try { + const state = createOwnershipReportState(); + const first = await runOwnershipReconciliationPass(f.db, { kinds, state }); + expect(first.kinds[0]).toMatchObject({ verdicts: { gone: 1 } }); + expect(recheck).toHaveBeenCalledWith([f.runId]); + // The next 30 s pass does not hand it over again; restart recovery schedules its own re-checks. + await runOwnershipReconciliationPass(f.db, { kinds, state }); + expect(recheck).toHaveBeenCalledTimes(1); + } finally { kill.mockRestore(); unregister(); } + }, 60_000); + it("keeps a live earlier-boot controller's result lease in the reconciler pass", async () => { const f = await seedAcceptedResult("ACCEPTED-RECONCILER-LIVE", (runner) => `${runner}:${randomUUID()}`); const self = await currentNativeControllerIdentity(); diff --git a/server/src/__tests__/ownership-reconciliation-report.test.ts b/server/src/__tests__/ownership-reconciliation-report.test.ts index bf4a4b371b4..72868565b8b 100644 --- a/server/src/__tests__/ownership-reconciliation-report.test.ts +++ b/server/src/__tests__/ownership-reconciliation-report.test.ts @@ -161,7 +161,7 @@ describe("report-only ownership reconciliation", () => { expect(new Set(names).size).toBe(names.length); expect(names.length).toBeGreaterThanOrEqual(40); expect(OWNABLE_KINDS.filter(kind => kind.settle !== null).map(kind => kind.id)) - .toEqual(["native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "sandbox_creation_intent", "environment_run_lease_active", "environment_teardown_claim", "environment_retained_orphan_sandbox", "environment_login_probe_lease", "workspace_runtime_control_operation", "native_workspace_finalize_attempt", "workspace_command_operation", "remote_snapshot_save", "execution_workspace_runtime_lease", "remote_workspace_unsaved_files", "remote_snapshot_ledger", "legacy_shutdown_settlement_fence", "copyback_owner_hold", "observed_workspace_preservation_fence", "wake_request_claimed", "legacy_controller_lease", "legacy_interrupted_preparation_hold", "native_completed_turn_cleanup_hold", "issue_execution_lock", "issue_checkout_lock", "agent_lifecycle_lease"]); + .toEqual(["native_executor_claim", "native_restart_recovery_claim", "native_awaiting_evidence", "native_accepted_result_lease", "native_finalizer_lease", "native_cleanup_lease", "sandbox_creation_intent", "environment_run_lease_active", "environment_teardown_claim", "environment_retained_orphan_sandbox", "environment_login_probe_lease", "workspace_runtime_control_operation", "native_workspace_finalize_attempt", "workspace_command_operation", "remote_snapshot_save", "execution_workspace_runtime_lease", "remote_workspace_unsaved_files", "remote_snapshot_ledger", "legacy_shutdown_settlement_fence", "copyback_owner_hold", "observed_workspace_preservation_fence", "wake_request_claimed", "legacy_controller_lease", "legacy_interrupted_preparation_hold", "native_completed_turn_cleanup_hold", "issue_execution_lock", "issue_checkout_lock", "agent_lifecycle_lease"]); expect(OWNABLE_KINDS.filter(kind => !kind.listInProgress).every(kind => kind.coverageGap)).toBe(true); }); diff --git a/server/src/index.ts b/server/src/index.ts index 76dbbbb71ca..c94ed99f2bc 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -92,6 +92,7 @@ import { registerNativeCleanupReleaseWake } from "./services/native-runtime/nati import { logNativeCleanupQuarantines } from "./services/native-runtime/native-cleanup-quarantine-log.js"; import { embeddedPostgresOwnerPort } from "./embedded-postgres-owner.js"; import { deliverExecutionStatuses } from "./services/execution-status-delivery.js"; +import { registerNativeRestartRecheck } from "./services/recovery/ownership-registry.js"; import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./services/execution-recovery-resolution.js"; import { deliverPendingPauseResumes } from "./services/issue-tree-resume.js"; import { deliverBudgetResumeContinuations, type BudgetResumeDelivery } from "./services/budget-resume-continuation.js"; @@ -962,6 +963,9 @@ async function startServerWithDatabaseTeardown( ? heartbeatService(db as any, { pluginWorkerManager }) : null; const accountingHeartbeat = heartbeat ?? heartbeatService(db as any, { pluginWorkerManager }); + // A native run whose controller the ownership reconciler proves gone is re-checked here, + // so a crashed old server's runs end or resume without another server boot. + if (heartbeat) registerNativeRestartRecheck(runIds => heartbeat.recoverNativeRunsAfterRestart(new Date(), runIds)); const decisionServiceOptions = { wakeOriginAgent: createDecisionWakeOriginAgent(heartbeat?.wakeup ?? null), }; diff --git a/server/src/services/controller-instance-lease.ts b/server/src/services/controller-instance-lease.ts index 08d6de7a790..b70eaa3dbc5 100644 --- a/server/src/services/controller-instance-lease.ts +++ b/server/src/services/controller-instance-lease.ts @@ -1,5 +1,5 @@ import { and, eq, isNull, sql } from "drizzle-orm"; -import { controllerInstances, type Db } from "@paperclipai/db"; +import { controllerInstanceLeaseDb, controllerInstanceLeaseDbWithoutStatementTimeout, controllerInstances, type Db } from "@paperclipai/db"; import { logger } from "../middleware/logger.js"; import { currentControllerBootId, currentProcessSpaceHash, onControllerStamp, @@ -20,6 +20,8 @@ const STAMP_COMMIT_GRACE_MS = CONTROLLER_LEASE_TTL_MS; const SETTLE_RECHECK_MIN_MS = 5_000; /** A renewal that has not answered within this bound counts as failed. */ const RENEW_TIMEOUT_MS = 10_000; +/** Server-side cancel for a renewal statement, below the client timeout. */ +export const RENEW_STATEMENT_TIMEOUT_MS = 8_000; const RETRY_MS = 5_000; export type ControllerInstanceState = "live" | "grace" | "expired" | "fenced" | "stopped"; @@ -95,14 +97,29 @@ function renewInDatabase(db: Db): (stampedRecently: boolean) => Promise lease.execute(sql` insert into controller_instances (boot_id, process_space, pid, lease_expires_at) values (${bootId}, ${currentProcessSpaceHash()}, ${process.pid}, ${expires}) on conflict (boot_id) do update set lease_expires_at = case when ${holds} then ${expires} else controller_instances.lease_expires_at end, renewed_at = case when ${holds} then now() else controller_instances.renewed_at end where controller_instances.fenced_at is null and controller_instances.stopped_at is null - returning epoch, ${holds} as holds, ${records} as records`) as unknown as Array<{ epoch: string | number; holds: boolean; records: boolean }>; + returning epoch, ${holds} as holds, ${records} as records`) as unknown as Promise>; + let rows: Awaited>; + try { + rows = await renew(controllerInstanceLeaseDb(db, { statementTimeoutMs: RENEW_STATEMENT_TIMEOUT_MS })); + } catch (error) { + // A connection pooler may refuse the startup parameter: renew without it. + if (!/unsupported startup parameter/i.test(error instanceof Error ? error.message : String(error))) throw error; + logger.warn({ err: error }, "controller instance lease connection refused statement_timeout; renewing without it"); + rows = await renew(controllerInstanceLeaseDbWithoutStatementTimeout(db)); + } const row = rows[0]; if (!row) return "fenced"; if (state) state.epoch = Number(row.epoch); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 6ddd401fcd9..ff976cc2d45 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -1433,11 +1433,29 @@ export async function claimNativeRestartRecoveries(input: { // old run's writes revoked and a new workspace root. // The same holds when the previous controller's host was never recorded // or cannot be proven from here: confirming its stop is optional. + // The run's own server boot is proven stopped or fenced (controller + // instance record): its lease is no sign of life, however long it was + // granted. The claim lease is 20 minutes for a local controller. + const ownerControllerStopped = evaluatedTakeover.allowed + && ["controller_stopped", "controller_fenced"].includes(evaluatedTakeover.reason); + const leaseLapsedAt = row.coordinator.leaseExpiresAt === null ? null + : row.coordinator.leaseExpiresAt <= now ? row.coordinator.leaseExpiresAt.getTime() : null; const unverifiableLapsedController = !takeover.allowed && !nonExecutionOwner - && (row.coordinator.leaseExpiresAt === null || row.coordinator.leaseExpiresAt <= now) + && (row.coordinator.leaseExpiresAt === null || leaseLapsedAt !== null || ownerControllerStopped) && (namespaceAttentionNeeded || ["controller_host_unrecorded", "controller_host_unprovable"].includes(takeover.reason)); + let unverifiedWaitEndsAt: number | null = null; if (unverifiableLapsedController) { - const waitingSince = namespaceWaitStartedAt(row.coordinator.recoveryHistory, takeover.reason); + // The bounded wait starts when the run's controller went away (its lease + // lapsed), not when a later server first looked: a successor that boots + // minutes later, or re-records the wait, must not restart the clock. + const observedSince = namespaceWaitStartedAt(row.coordinator.recoveryHistory, takeover.reason); + const starts = [observedSince, leaseLapsedAt].filter((at): at is number => at !== null); + // A controller boot proven stopped or fenced already outlived its lease + // (90 s) and the fence grace (30 s): waiting longer proves nothing more + // about a Runner in another process space, so the run ends now. + const waitingSince = ownerControllerStopped ? now.getTime() - NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS + : starts.length ? Math.min(...starts) : null; + unverifiedWaitEndsAt = (waitingSince ?? now.getTime()) + NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS; if (waitingSince !== null && now.getTime() - waitingSince >= NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS) { await endNamespaceUnverifiedRun(tx as unknown as Db, row, now, takeover.reason, historyEntry({ now, restartKind: input.restartKind, controller, generation: row.coordinator.controllerGeneration, @@ -1480,8 +1498,12 @@ export async function claimNativeRestartRecoveries(input: { // Repeated scans of one ownership wait record it once, so the // bounded history keeps the drain record the next claim reads. ...(waitAlreadyRecorded ? {} : { recoveryHistory: appendBoundedRecoveryHistory(event) }), - updatedAt: waitAlreadyRecorded && row.coordinator.recoveryState === "awaiting_evidence" - && row.coordinator.recoveryRequestId === (input.recoveryRequestId ?? null) ? row.coordinator.updatedAt : now, + // updated_at is the owner's last sign of life: the ownership prover + // fences a dead boot only once it has been quiet past that write plus + // the lease and grace, and the fallback reads it as the owner's + // progress. A successor recording its own wait never moves it. + updatedAt: !ownControllerProcess || (waitAlreadyRecorded && row.coordinator.recoveryState === "awaiting_evidence" + && row.coordinator.recoveryRequestId === (input.recoveryRequestId ?? null)) ? row.coordinator.updatedAt : now, }) .where( and( @@ -1502,7 +1524,8 @@ export async function claimNativeRestartRecoveries(input: { : ["controller_lease_held", "controller_lease_grace"].includes(takeover.reason) ? { retryAt: new Date(now.getTime() + CONTROLLER_LEASE_RENEW_MS) } // The bounded wait for an unverifiable controller is checked again when it ends. - : unverifiableLapsedController ? { retryAt: new Date(now.getTime() + NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS) } : {}), + : unverifiableLapsedController ? { retryAt: new Date(Math.max(now.getTime() + 1_000, + unverifiedWaitEndsAt ?? now.getTime() + NATIVE_NAMESPACE_UNVERIFIED_WAIT_MS)) } : {}), } as const; } diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 04152a91dac..9a01087d3e5 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -2478,18 +2478,23 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); const [run] = await fixture.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, fixture.runId)); const stopped = await hasNativeLocalProcessStop(fixture.db, companyId, fixture.runId); expect(probes).toEqual([]); - // The namespace wait is bounded: it is checked again when the wait ends. - expect(dispositions).toEqual([{ kind: "awaiting_evidence", runId: fixture.runId, - reason: "controller_process_space_unverified", retryAt: expect.any(Date) }]); + // A stopped or fenced controller boot already outlived its lease and the + // fence grace, so the namespace wait ends on the first look: the run ends + // as unverified (never as a proven stop) and the task continues in a fresh + // run. Nothing local is probed, claimed or recorded as stopped. + expect(dispositions).toEqual([{ kind: "blocked", runId: fixture.runId, + reason: "controller_process_space_unverified" }]); expect(claimed).not.toHaveBeenCalled(); expect(stamped).not.toHaveBeenCalled(); expect(stopped).toBe(false); - expect(after).toMatchObject({ phase: before.phase, attempt: before.attempt, + expect(after).toMatchObject({ phase: "terminal_failure", attempt: before.attempt, controllerGeneration: before.controllerGeneration, controllerBootId: before.controllerBootId, controllerPid: before.controllerPid, controllerHostHash: before.controllerHostHash, - leaseOwner: before.leaseOwner, leaseExpiresAt: before.leaseExpiresAt, resultId: null }); - expect(run).toMatchObject({ status: "running", processPid: runnerPid, processGroupId: runnerGroup, - processStartedAt: startedAt, processHostHash: foreignSpace }); + failureCode: "native_restart_recovery_blocked", + failureDetail: expect.objectContaining({ reason: "controller_process_space_unverified", executorStopEvidence: "unverified" }), + resultId: null }); + expect(run).toMatchObject({ status: "failed", errorCode: "native_restart_recovery_blocked", + processPid: runnerPid, processGroupId: runnerGroup, processStartedAt: startedAt, processHostHash: foreignSpace }); expect(run.runnerProfileJson).toEqual({ ...fixture.preparedProfile, sessionCheckpoint: { identity: { companyId, issueId: fixture.issueId, runId: fixture.runId, agentId, sessionId: fixture.native.normalizedSessionId }, diff --git a/server/src/services/recovery/ownership-registry.ts b/server/src/services/recovery/ownership-registry.ts index 27be9c93c9c..66c5f07b43f 100644 --- a/server/src/services/recovery/ownership-registry.ts +++ b/server/src/services/recovery/ownership-registry.ts @@ -275,6 +275,37 @@ function nrfKind(id: string, title: string, obligation: Obligation, where: SQL | }; } +/** + * A native run whose controller died after another server's startup pass + * (a crash mid-overlap, not a graceful drain) has no server re-checking it: + * restart recovery only schedules re-checks for runs it saw. When the + * reconciler proves such a run's owner gone, this server runs the same restart + * recovery for that run now; its bounded wait and re-check schedule then end + * or resume the run without waiting for another server to boot. + */ +let nativeRestartRecheck: ((runIds: string[]) => Promise) | null = null; +export function registerNativeRestartRecheck(recheck: (runIds: string[]) => Promise) { + nativeRestartRecheck = recheck; + return () => { if (nativeRestartRecheck === recheck) nativeRestartRecheck = null; }; +} +/** The reconciler re-checks every 30 s; restart recovery schedules its own + * follow-ups, so one hand-off per run per minute is enough. */ +const NATIVE_RESTART_RECHECK_SPACING_MS = 60_000; +const lastNativeRestartRecheck = new Map(); +const recheckNativeRunOwnership: SettleRecord = async (_db, record, context) => { + if (!nativeRestartRecheck || !record.runId) return { settled: false, reason: "restart_recheck_unavailable" }; + const last = lastNativeRestartRecheck.get(record.runId); + if (last !== undefined && context.now.getTime() - last < NATIVE_RESTART_RECHECK_SPACING_MS) + return { settled: false, reason: "restart_recovery_recheck_recent" }; + if (lastNativeRestartRecheck.size > 10_000) lastNativeRestartRecheck.clear(); + lastNativeRestartRecheck.set(record.runId, context.now.getTime()); + await nativeRestartRecheck([record.runId]); + return { settled: false, reason: "restart_recovery_rechecked" }; +}; +function withNativeRestartRecheck(kind: OwnableKind): OwnableKind { + return { ...kind, settle: recheckNativeRunOwnership }; +} + const notFinalizerOrCleanup = and(notLike(nativeRunFinalizations.leaseOwner, "native-finalizer:%"), notLike(nativeRunFinalizations.leaseOwner, "native-cleanup:%")); @@ -682,13 +713,13 @@ const acceptedNativeResultLease = nrfKind("native_accepted_result_lease", "Accep export const OWNABLE_KINDS: readonly OwnableKind[] = [ // Native runtime and Runner - nrfKind("native_executor_claim", "Executor execution claim (fresh run)", "ownership", + withNativeRestartRecheck(nrfKind("native_executor_claim", "Executor execution claim (fresh run)", "ownership", and(eq(nativeRunFinalizations.phase, "observed"), isNull(nativeRunFinalizations.resultId), - isNotNull(nativeRunFinalizations.leaseOwner), notFinalizerOrCleanup, isNull(nativeRunFinalizations.recoveryState))), - nrfKind("native_restart_recovery_claim", "Restart-recovery claim", "ownership", - inArray(nativeRunFinalizations.recoveryState, ["awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"])), - nrfKind("native_awaiting_evidence", "\"Awaiting evidence\" wait", "ownership", - eq(nativeRunFinalizations.recoveryState, "awaiting_evidence"), undefined, [1, 2, 7]), + isNotNull(nativeRunFinalizations.leaseOwner), notFinalizerOrCleanup, isNull(nativeRunFinalizations.recoveryState)))), + withNativeRestartRecheck(nrfKind("native_restart_recovery_claim", "Restart-recovery claim", "ownership", + inArray(nativeRunFinalizations.recoveryState, ["awaiting_runner_reattach", "resuming_session", "bootstrap_incomplete"]))), + withNativeRestartRecheck(nrfKind("native_awaiting_evidence", "\"Awaiting evidence\" wait", "ownership", + eq(nativeRunFinalizations.recoveryState, "awaiting_evidence"), undefined, [1, 2, 7])), { // Shared path since Phase 2: a hard kill after `run.result` is accepted // leaves the result leased to the dead controller; settle expires that