diff --git a/server/src/__tests__/live-recovery-work.test.ts b/server/src/__tests__/live-recovery-work.test.ts index 1e78c73c160..495b581de55 100644 --- a/server/src/__tests__/live-recovery-work.test.ts +++ b/server/src/__tests__/live-recovery-work.test.ts @@ -6,7 +6,7 @@ import { once } from "node:events"; import { setImmediate as immediate, setTimeout as realDelay } from "node:timers/promises"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, authUsers, companies, companyMemberships, createDb, closeLivenessDb, closeCheckpointDb, +import { agents, authUsers, companies, companyMemberships, createDb, closeLivenessDb, closeCheckpointDb, livenessDb, heartbeatRuns, heartbeatRunEvents, nativeRunFinalizations, environmentLeases, issueThreadInteractions, applyPendingMigrations, issueRecoveryActions, issues, activityLog, agentWakeupRequests, reconcileDatabaseWork } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; @@ -35,6 +35,7 @@ import { nativeRetryCancellationEligible, pendingFailedNativeRetryCancellationCo import { requestNativeQuestionRunCancellation } from "../services/native-runtime/native-question-bridge.js"; import { readNativeLocalProcessStop, releaseSettledNativeRunProcess, settleNativeRunProcess } from "../services/native-local-process-stop.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { renewNativeSessionExecutionLease } from "../services/native-runtime/native-execution-lease.js"; import { buildNativeExecutionInput } from "../services/native-runtime/native-execution-input.js"; import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js"; import * as nativeStopEvidence from "../services/acknowledged-native-stop.js"; @@ -580,6 +581,63 @@ describe("residual recovery production matrix", () => { await heartbeat.cancelRun(run.id, "Operator cancelled the controller renewal fixture"); await quiet(); }); + it("renews a native execution lease without scheduling recovery and still finds the run at the end of its last lease", async () => { + // Another server owns the run. Its renewals must not make this server's + // recovery re-check the healthy run, and when they stop, recovery acts at + // the expiry of the last renewed lease, as it would have before. + const f = await fixture(), old = new Date(Date.now() - 6 * 60_000); + const owner = { bootId: randomUUID(), pid: process.pid + 100_000, processStartedAt: old }; + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, + status: "running", runtimeMode: "native", nativeIssueId: f.issue.id, contextSnapshot: { issueId: f.issue.id }, + startedAt: old, updatedAt: old }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id, + phase: "observed", attempt: 1, leaseOwner: "other-server", controllerBootId: owner.bootId, controllerPid: owner.pid, + controllerProcessStartedAt: old, leaseExpiresAt: sql`clock_timestamp() + interval '3 seconds'` }); + const savedExpiry = async () => (await db.select().from(nativeRunFinalizations) + .where(eq(nativeRunFinalizations.runId, run.id)))[0].leaseExpiresAt!.getTime(); + const expiry = await savedExpiry(); + await start(true); + await vi.waitFor(() => expect(deadlines.get("active-execution-recovery")!()).toBe(expiry)); + await Promise.allSettled([...passes]); + const before = timeline.length; + // Renew through this client so a recovery signal, if any, reaches the + // registered owners in-process, as it reaches peers through notifications. + try { + await renewNativeSessionExecutionLease({ db, runId: run.id, companyId: f.companyId, issueId: f.issue.id, + leaseOwner: "other-server", attempt: 1, controller: owner, leaseTtlMs: 6_000 }); + } finally { + // Only this test uses the private liveness pool; end it while timers are real. + await livenessDb(db).$client.end({ timeout: 1 }); + } + const moved = await savedExpiry(); + expect(moved).toBeGreaterThan(expiry); + await realDelay(1_000); + expect(timeline.slice(before).filter(event => Array.isArray(event) && event[0] === "pass")).toEqual([]); + // At the old deadline recovery re-reads the row and moves to the new one. + await realDelay(Math.max(0, expiry - Date.now() + 500)); + await Promise.allSettled([...passes]); + expect(await getRun(run.id)).toMatchObject({ status: "running", errorCode: null }); + expect(deadlines.get("active-execution-recovery")!()).toBe(moved); + // The owner stopped renewing: recovery acts at the end of the last lease. + const acted = async () => { + const current = await getRun(run.id); + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); + return current.status !== "running" || current.errorCode !== null || coordinator.recoveryState !== null + || coordinator.leaseOwner !== "other-server"; + }; + expect(await acted()).toBe(false); + await vi.waitFor(async () => expect(await acted()).toBe(true), { timeout: 15_000 }); + expect(Date.now()).toBeGreaterThanOrEqual(moved); + expect(Date.now() - moved).toBeLessThan(10_000); + await Promise.allSettled([...passes]); + await db.transaction(async tx => { + await notifyExecutionRunWork(tx, { companyId: f.companyId, runId: run.id, agentId: f.agent.id }); + await tx.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", recoveryState: "blocked", leaseExpiresAt: null }) + .where(eq(nativeRunFinalizations.runId, run.id)); + }); + await quiet(); + }); it("recovers orphan active leases after restart, including a deleted heartbeat reference, without starving behind 100 guarded rows", async () => { const f = await fixture(), foreign = await fixture(), resource = randomUUID(), old = new Date(Date.now() - 6 * 60_000); await db.insert(environmentLeases).values({ companyId: foreign.companyId, status: "retained", provider: "fixture", providerLeaseId: resource, updatedAt: old }); diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index bf894afacab..f1336e9c1ad 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -15,7 +15,7 @@ import type { HotRestartIntent } from "../hot-restart.js"; import type { NativeRestartRecoveryClaim, NativeRestartRecoveryDisposition } from "../native-runtime/index.js"; import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "../native-runtime/native-runner-ownership.js"; import { claimNativeRestartRecoveries, currentNativeControllerIdentity, NATIVE_RESTART_RECOVERY_FAILED } from "../native-runtime/native-restart-recovery.js"; -import { startNativeSessionExecutionLeaseRenewal } from "../native-runtime/native-execution-lease.js"; +import { renewNativeExecutionLeasesNow, startNativeSessionExecutionLeaseRenewal } from "../native-runtime/native-execution-lease.js"; import { createRunLogWriterAuthority } from "./run-log-authority.js"; import { allowLegacyShutdownWorkspaceCleanup, beginLegacyShutdownWorkspaceSettlement, finishLegacyShutdownWorkspaceSettlement, maintainLegacyShutdownWorkspaceSettlement, legacyShutdownWorkspaceResourceProtected, sweepLegacyShutdownWorkspaceSettlements } from "../legacy-shutdown-workspace-settlement.js"; import { adapterExecutionControls, createAdapterExecutionControl } from "../adapter-execution-control.js"; @@ -1367,7 +1367,6 @@ describePostgres("heartbeat recovery module ownership", () => { await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, phase: "observed", attempt: 1, controllerBootId: controller.bootId, controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt, leaseOwner: claim.leaseOwner, leaseExpiresAt: firstLeaseEndsAt }); - vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); native.claims.mockImplementationOnce(async input => { await input.onClaim?.(claim, scope); return [claim]; @@ -1378,7 +1377,8 @@ describePostgres("heartbeat recovery module ownership", () => { const outcome = new Promise(resolve => { report = resolve; }); const finished = new Promise(resolve => { finish = resolve; }); deps.executeRun.mockImplementationOnce(async (_runId, options) => { try { - await vi.advanceTimersByTimeAsync(21_000); + // The shared renewal timer renews the restart claim (preparation). + await renewNativeExecutionLeasesNow(db); await vi.waitFor(async () => { const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); expect(coordinator.leaseExpiresAt!.getTime()).toBeGreaterThan(firstLeaseEndsAt.getTime() + 10_000); @@ -1391,7 +1391,7 @@ describePostgres("heartbeat recovery module ownership", () => { try { await options?.onNativeExecutionLeaseAcquired?.(); const [before] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); - await vi.advanceTimersByTimeAsync(21_000); + await renewNativeExecutionLeasesNow(db); await vi.waitFor(async () => { const [after] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); expect(after.leaseExpiresAt!.getTime()).toBeGreaterThan(before.leaseExpiresAt!.getTime()); @@ -1410,7 +1410,6 @@ describePostgres("heartbeat recovery module ownership", () => { } finally { finish(); await Promise.all([...deps.activeRunExecutionPromises]); - vi.useRealTimers(); await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); } diff --git a/server/src/services/native-runtime/native-execution-lease-batch.test.ts b/server/src/services/native-runtime/native-execution-lease-batch.test.ts new file mode 100644 index 00000000000..81ab40ab522 --- /dev/null +++ b/server/src/services/native-runtime/native-execution-lease-batch.test.ts @@ -0,0 +1,292 @@ +import { randomUUID } from "node:crypto"; +import { eq, inArray, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, closeLivenessDb, companies, createDb, heartbeatRuns, issues, livenessDb, nativeRunFinalizations, +} from "@paperclipai/db"; +import { + renewNativeExecutionLeasesNow, startNativeSessionExecutionLeaseRenewal, type NativeExecutionLeaseRenewal, +} from "./native-execution-lease.js"; +import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; +import { meterDatabaseStatements } from "../../__tests__/helpers/database-statement-meter.js"; +import { + getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, +} from "../../__tests__/helpers/embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = support.supported ? describe : describe.skip; +if (!support.supported) console.warn(`Skipping batched native lease renewal tests: ${support.reason ?? "unsupported environment"}`); + +const controller = { bootId: `boot-${randomUUID()}`, pid: 5151, processStartedAt: new Date("2026-11-13T08:00:00.000Z") }; + +/** + * One server process renews the native execution leases of all its runs in + * one shared transaction per tick, on the private two-connection liveness pool. + */ +describeEmbeddedPostgres("batched native execution lease renewal", () => { + let db!: ReturnType; + let owner!: ReturnType; + let meter!: ReturnType; + let connectionString!: string; + let stopDatabase: (() => Promise) | undefined; + let companyId!: string; + let agentId!: string; + + beforeAll(async () => { + const started = await startEmbeddedPostgresTestDatabase("paperclip-native-lease-batch-"); + stopDatabase = started.cleanup; + connectionString = started.connectionString; + db = createDb(connectionString); + owner = createDb(connectionString); + meter = meterDatabaseStatements(livenessDb(owner), connectionString); + companyId = randomUUID(); + agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Lease batch", issuePrefix: "LBT", + requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Runner", role: "engineer", status: "active", + adapterType: "paperclip_runner", adapterConfig: { provider: "codex" }, runtimeConfig: {}, permissions: {} }); + }, 120_000); + + afterAll(async () => { + if (owner) await closeLivenessDb(owner, 1).catch(() => undefined); + await owner?.$client.end({ timeout: 1 }); + await db?.$client.end({ timeout: 1 }); + await stopDatabase?.(); + }); + + type Seeded = { runId: string; issueId: string; leaseOwner: string }; + + async function seedRuns(count: number, leaseMs = 30_000): Promise { + const runs = Array.from({ length: count }, () => ({ runId: randomUUID(), issueId: randomUUID(), + leaseOwner: `runner:${randomUUID()}` })); + await db.insert(issues).values(runs.map(run => ({ id: run.issueId, companyId, identifier: `LBT-${run.runId.slice(0, 8)}`, + title: "batch", status: "in_progress", priority: "medium", workMode: "standard", assigneeAgentId: agentId }))); + await db.insert(heartbeatRuns).values(runs.map(run => ({ id: run.runId, companyId, agentId, status: "running", + runtimeMode: "native", nativeIssueId: run.issueId, invocationSource: "assignment", triggerDetail: "system", + contextSnapshot: { issueId: run.issueId } }))); + await db.insert(nativeRunFinalizations).values(runs.map(run => ({ runId: run.runId, companyId, issueId: run.issueId, + phase: "observed", attempt: 1, leaseOwner: run.leaseOwner, + leaseExpiresAt: sql`clock_timestamp() + (${leaseMs} * interval '1 millisecond')`, + controllerBootId: controller.bootId, controllerPid: controller.pid, + controllerProcessStartedAt: controller.processStartedAt, controllerGeneration: 1 }))); + return runs; + } + + function start(run: Seeded, overrides: { remote?: boolean; controllerGeneration?: number } = {}) { + const onLost = vi.fn(async () => undefined); + const guard = startNativeSessionExecutionLeaseRenewal({ db: owner, companyId, ...run, attempt: 1, controller, + remote: overrides.remote ?? true, controllerGeneration: overrides.controllerGeneration, + confirmedDeadline: performance.now() + 25_000, onLost }); + return { run, guard, onLost }; + } + + async function expiries(runIds: string[]) { + const rows = await db.select({ runId: nativeRunFinalizations.runId, expiresAt: nativeRunFinalizations.leaseExpiresAt, + leaseOwner: nativeRunFinalizations.leaseOwner }) + .from(nativeRunFinalizations).where(inArray(nativeRunFinalizations.runId, runIds)); + return new Map(rows.map(row => [row.runId, row])); + } + + async function stopAll(guards: NativeExecutionLeaseRenewal[]) { + await Promise.all(guards.map(guard => guard.stop().catch(() => undefined))); + } + + async function renewCost(count: number) { + const runs = await seedRuns(count); + const started = runs.map(run => start(run)); + try { + // Warm the pool so connection setup is not counted. + await renewNativeExecutionLeasesNow(owner); + const before = meter.read(); + await renewNativeExecutionLeasesNow(owner); + const after = meter.read(); + for (const { guard, onLost } of started) { + expect(onLost).not.toHaveBeenCalled(); + expect(guard.remainingMs()).toBeGreaterThan(REMOTE_CONTROLLER_LEASE_MS - 5_000); + } + const saved = await expiries(runs.map(run => run.runId)); + for (const run of runs) expect(saved.get(run.runId)!.expiresAt!.getTime()).toBeGreaterThan(Date.now() + 50_000); + return { transactions: after.transactions - before.transactions, statements: after.statements - before.statements }; + } finally { + await stopAll(started.map(item => item.guard)); + } + } + + it("renews every lease of the process in one transaction whose size does not grow with the run count", async () => { + const few = await renewCost(3); + const many = await renewCost(60); + expect(few.transactions).toBe(1); + expect(many.transactions).toBe(1); + expect(many.statements).toBe(few.statements); + }, 60_000); + + it("splits a very large tick into bounded transactions", async () => { + const cost = await renewCost(450); + expect(cost.transactions).toBe(3); + }, 120_000); + + it("fails only the runs whose lease is no longer theirs and renews the rest", async () => { + const runs = await seedRuns(6); + const [healthy, other, takenOver, newAttempt, otherBoot, expired] = runs; + await db.update(nativeRunFinalizations).set({ leaseOwner: `native-finalizer:${randomUUID()}` }) + .where(eq(nativeRunFinalizations.runId, takenOver!.runId)); + await db.update(nativeRunFinalizations).set({ attempt: 2 }).where(eq(nativeRunFinalizations.runId, newAttempt!.runId)); + await db.update(nativeRunFinalizations).set({ controllerBootId: `boot-${randomUUID()}` }) + .where(eq(nativeRunFinalizations.runId, otherBoot!.runId)); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: sql`clock_timestamp() - interval '1 second'` }) + .where(eq(nativeRunFinalizations.runId, expired!.runId)); + const before = await expiries(runs.map(run => run.runId)); + const started = runs.map(run => start(run)); + try { + await renewNativeExecutionLeasesNow(owner); + // A newer attempt under the same owner is the owner's own executor + // moving on: not extended and not lost, as for a single renewal. + const [ok, ok2, takenOverRenewal, advanced, otherBootRenewal, expiredRenewal] = started; + const lost = [takenOverRenewal!, otherBootRenewal!, expiredRenewal!]; + expect(advanced!.onLost).not.toHaveBeenCalled(); + for (const item of [ok!, ok2!]) { + expect(item.onLost).not.toHaveBeenCalled(); + await expect(item.guard.stop()).resolves.toBeUndefined(); + } + for (const item of lost) { + await vi.waitFor(() => expect(item.onLost).toHaveBeenCalledTimes(1)); + await expect(item.guard.stop()).rejects.toThrow("native_session_lease_lost"); + } + const after = await expiries(runs.map(run => run.runId)); + for (const run of [healthy!, other!]) { + expect(after.get(run.runId)!.expiresAt!.getTime()).toBeGreaterThan(before.get(run.runId)!.expiresAt!.getTime()); + } + // Nothing the batch could not extend was written, and the expired owner stays expired. + for (const run of [takenOver!, newAttempt!, otherBoot!, expired!]) expect(after.get(run.runId)).toEqual(before.get(run.runId)); + } finally { + await stopAll(started.map(item => item.guard)); + } + }, 60_000); + + it("never revives an owner that expired while another writer held its row, and does not wait for that row", async () => { + const [contended, healthy] = await seedRuns(2, 1_500); + // Another actor (for example restart recovery) holds the coordinator row + // from before the lease expires until after it expired, then commits. + let release!: () => void; + let locked!: () => void; + const held = new Promise(resolve => { release = resolve; }); + const acquired = new Promise(resolve => { locked = resolve; }); + const writer = db.transaction(async tx => { + await tx.execute(sql`select 1 from ${nativeRunFinalizations} where ${nativeRunFinalizations.runId} = ${contended!.runId} for update`); + locked(); + await held; + }); + await acquired; + const contendedRenewal = start(contended!); + const healthyRenewal = start(healthy!); + try { + const renewing = renewNativeExecutionLeasesNow(owner); + // The busy row does not hold up the rest of the batch. + await vi.waitFor(async () => expect((await expiries([healthy!.runId])).get(healthy!.runId)!.expiresAt!.getTime()) + .toBeGreaterThan(Date.now() + 50_000), { timeout: 5_000 }); + await new Promise(resolve => setTimeout(resolve, 2_000)); + expect((await expiries([contended!.runId])).get(contended!.runId)!.expiresAt!.getTime()).toBeLessThan(Date.now()); + release(); + await writer; + await renewing; + await vi.waitFor(() => expect(contendedRenewal.onLost).toHaveBeenCalledTimes(1)); + await expect(contendedRenewal.guard.stop()).rejects.toThrow("native_session_lease_lost"); + expect((await expiries([contended!.runId])).get(contended!.runId)!.expiresAt!.getTime()).toBeLessThan(Date.now()); + expect(healthyRenewal.onLost).not.toHaveBeenCalled(); + } finally { + release(); + await writer.catch(() => undefined); + await stopAll([contendedRenewal.guard, healthyRenewal.guard]); + } + }, 60_000); + + it("never revives an owner whose row a takeover rewrote while the batch waited", async () => { + const [run] = await seedRuns(1, 60_000); + const renewal = start(run!); + // A takeover commits between the batch's statements on another connection; + // the batch locks the row and reads its committed owner, so it extends nothing. + await db.update(nativeRunFinalizations).set({ leaseOwner: `native-cleanup:${randomUUID()}`, + leaseExpiresAt: sql`clock_timestamp() + interval '5 minutes'` }).where(eq(nativeRunFinalizations.runId, run!.runId)); + const before = await expiries([run!.runId]); + try { + await renewNativeExecutionLeasesNow(owner); + await vi.waitFor(() => expect(renewal.onLost).toHaveBeenCalledTimes(1)); + expect(await expiries([run!.runId])).toEqual(before); + } finally { + await renewal.guard.stop().catch(() => undefined); + } + }, 60_000); + + it("confines a binding that makes the shared statement fail to its own run", async () => { + const runs = await seedRuns(5); + const before = await expiries(runs.map(run => run.runId)); + const healthy = runs.slice(1).map(run => start(run)); + // An attempt outside the integer range: PostgreSQL rejects the whole + // shared UPDATE (22003) for every run in it. + const poison = { run: runs[0]!, onLost: vi.fn(async () => undefined), guard: startNativeSessionExecutionLeaseRenewal({ + db: owner, companyId, ...runs[0]!, attempt: 2 ** 40, controller, remote: true, + confirmedDeadline: performance.now() + 25_000, onLost: async () => { await poison.onLost(); } }) }; + try { + await renewNativeExecutionLeasesNow(owner); + const after = await expiries(runs.map(run => run.runId)); + for (const { run, onLost, guard } of healthy) { + expect(onLost).not.toHaveBeenCalled(); + expect(after.get(run.runId)!.expiresAt!.getTime()).toBeGreaterThan(before.get(run.runId)!.expiresAt!.getTime()); + expect(guard.remainingMs()).toBeGreaterThan(REMOTE_CONTROLLER_LEASE_MS - 5_000); + } + expect(after.get(poison.run.runId)).toEqual(before.get(poison.run.runId)); + // The healthy runs go back to one shared transaction; nothing is stuck busy. + const cost = meter.read(); + await renewNativeExecutionLeasesNow(owner); + const healthyCost = meter.read(); + expect(healthyCost.transactions - cost.transactions).toBe(1); + for (const { onLost } of healthy) expect(onLost).not.toHaveBeenCalled(); + // Only the bad binding's own run is lost. + await vi.waitFor(() => expect(poison.onLost).toHaveBeenCalledTimes(1)); + } finally { + await stopAll([...healthy.map(item => item.guard), poison.guard]); + } + }, 60_000); + + it("handles an error thrown while the shared statement is built and leaves no lease busy", async () => { + const runs = await seedRuns(3); + const healthy = runs.slice(1).map(run => start(run)); + const onLost = vi.fn(async () => undefined); + // An invalid date throws in toISOString() while the VALUES list is built. + const broken = startNativeSessionExecutionLeaseRenewal({ db: owner, companyId, ...runs[0]!, attempt: 1, + controller: { ...controller, processStartedAt: new Date(Number.NaN) }, remote: true, + confirmedDeadline: performance.now() + 25_000, onLost }); + try { + await renewNativeExecutionLeasesNow(owner); + const after = await expiries(runs.map(run => run.runId)); + for (const { run, onLost: healthyLost } of healthy) { + expect(healthyLost).not.toHaveBeenCalled(); + expect(after.get(run.runId)!.expiresAt!.getTime()).toBeGreaterThan(Date.now() + 50_000); + } + // Every lease settled: a second call renews again instead of finding them busy. + const before = meter.read(); + await renewNativeExecutionLeasesNow(owner); + expect(meter.read().transactions - before.transactions).toBe(1); + for (const { guard } of healthy) expect(guard.remainingMs()).toBeGreaterThan(REMOTE_CONTROLLER_LEASE_MS - 5_000); + await vi.waitFor(() => expect(onLost).toHaveBeenCalledTimes(1)); + } finally { + await stopAll([...healthy.map(item => item.guard), broken]); + } + }, 60_000); + + it("renews local and remote leases together, each for its own lease length", async () => { + const [remote, local] = await seedRuns(2); + const started = [start(remote!), start(local!, { remote: false, controllerGeneration: 1 })]; + try { + const before = meter.read(); + await renewNativeExecutionLeasesNow(owner); + expect(meter.read().transactions - before.transactions).toBe(1); + const saved = await expiries([remote!.runId, local!.runId]); + expect(saved.get(remote!.runId)!.expiresAt!.getTime()).toBeLessThan(Date.now() + 2 * 60_000); + expect(saved.get(local!.runId)!.expiresAt!.getTime()).toBeGreaterThan(Date.now() + 15 * 60_000); + expect(started[1]!.guard.remainingMs()).toBeGreaterThan(15 * 60_000); + } finally { + await stopAll(started.map(item => item.guard)); + } + }, 60_000); +}); diff --git a/server/src/services/native-runtime/native-execution-lease-reacquire.test.ts b/server/src/services/native-runtime/native-execution-lease-reacquire.test.ts index a5989b6802b..5fd94b4cb96 100644 --- a/server/src/services/native-runtime/native-execution-lease-reacquire.test.ts +++ b/server/src/services/native-runtime/native-execution-lease-reacquire.test.ts @@ -5,8 +5,8 @@ import { agents, closeLivenessDb, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, } from "@paperclipai/db"; import { - lockNativeExecutionLeaseForReacquire, nativeExecutionLeaseHeldCondition, renewNativeSessionExecutionLease, - startNativeSessionExecutionLeaseRenewal, + lockNativeExecutionLeaseForReacquire, nativeExecutionLeaseHeldCondition, renewNativeExecutionLeasesNow, + renewNativeSessionExecutionLease, startNativeSessionExecutionLeaseRenewal, } from "./native-execution-lease.js"; import { OPERATOR_ATTESTED_CONTROLLER_HOST } from "./native-controller-host-recovery.js"; import { NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE } from "./native-runner-ownership.js"; @@ -90,13 +90,14 @@ describeEmbeddedPostgres("native execution lease taken back after a host sleep", it("renews from the shared timer after the sleep instead of interrupting the turn", async () => { const run = await seedSleptRun("timer"); - vi.useFakeTimers({ toFake: ["setInterval"] }); const onLost = vi.fn(async () => undefined); const { reacquire: _reacquire, ...input } = run.renewal; const guard = startNativeSessionExecutionLeaseRenewal({ ...input, remote: false, controllerGeneration: GENERATION, onLost }); try { - await vi.advanceTimersByTimeAsync(5 * 60_000); + // The shared batch finds the lapsed lease and hands it to the single-run + // path, which takes it back under the run row lock. + await renewNativeExecutionLeasesNow(owner); await vi.waitFor(async () => expect((await row(run.runId)).leaseExpiresAt!.getTime()).toBeGreaterThan(Date.now())); expect(onLost).not.toHaveBeenCalled(); } finally { @@ -125,11 +126,10 @@ describeEmbeddedPostgres("native execution lease taken back after a host sleep", await expect(renewNativeSessionExecutionLease(legacy)).rejects.toThrow("native_session_lease_lost"); await expect(renewNativeSessionExecutionLease({ ...legacy, reacquire: { controllerGeneration: GENERATION + 1 } })) .rejects.toThrow("native_session_lease_lost"); - vi.useFakeTimers({ toFake: ["setInterval"] }); const onLost = vi.fn(async () => undefined); const guard = startNativeSessionExecutionLeaseRenewal({ ...legacy, remote: true, controllerGeneration: GENERATION, onLost }); - await vi.advanceTimersByTimeAsync(60_000); + await renewNativeExecutionLeasesNow(owner); await vi.waitFor(() => expect(onLost).toHaveBeenCalled()); await expect(guard.stop()).rejects.toThrow("native_session_lease_lost"); expect((await row(run.runId)).leaseExpiresAt!.getTime()).toBeLessThan(Date.now()); diff --git a/server/src/services/native-runtime/native-execution-lease.test.ts b/server/src/services/native-runtime/native-execution-lease.test.ts index 9b5f97c5caf..26145d855f5 100644 --- a/server/src/services/native-runtime/native-execution-lease.test.ts +++ b/server/src/services/native-runtime/native-execution-lease.test.ts @@ -1,16 +1,29 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; import { startNativeSessionExecutionLeaseRenewal } from "./native-execution-lease.js"; +const dialect = new PgDialect(); + function fixture() { + // `returning` decides each renewal: the batched UPDATE and the single-run + // UPDATE both resolve through it. const returning = vi.fn(async () => [{ runId: "run" }]); const coordinator = vi.fn(async (): Promise> => []); const renewed = vi.fn((_values: unknown) => undefined); const update = vi.fn(() => ({ set: (values: unknown) => { renewed(values); return { where: (condition: unknown) => { renewed(condition); return { returning }; } }; } })); const select = () => ({ from: () => ({ where: () => ({ limit: coordinator }) }) }); - const transaction = vi.fn(async (run: (tx: unknown) => Promise) => run({ update, select, execute: vi.fn() })); + // The batch locks the coordinator row first; `lock` is what that lock reads. + const lock = { present: true, live: true }; + const execute = vi.fn(async (query: SQL) => { + const text = dialect.sqlToQuery(query).sql; if (text.includes("skip locked")) return lock.present ? [{ run_id: "run", live: lock.live }] : []; + if (text.includes("set lease_expires_at")) return (await returning()).length ? [{ run_id: "run" }] : []; + return []; + }); + const transaction = vi.fn(async (run: (tx: unknown) => Promise) => run({ update, select, execute })); const db = { update, transaction }; - return { update, returning, transaction, coordinator, input: { db: db as unknown as Db, runId: "run", companyId: "company", issueId: "issue", + return { update, returning, transaction, coordinator, lock, input: { db: db as unknown as Db, runId: "run", companyId: "company", issueId: "issue", leaseOwner: "owner", attempt: 1, controller: { bootId: "boot", pid: 1, processStartedAt: new Date() }, remote: true, onLost: vi.fn(async () => undefined) } }; } @@ -32,7 +45,7 @@ describe("native execution renewal ownership", () => { }); it("keeps a successfully renewed lease through repeated slow connection acquisition", async () => { vi.useFakeTimers(); - const { input, update, transaction } = fixture(); + const { input, returning, transaction } = fixture(); const connected = transaction.getMockImplementation()!; transaction.mockImplementation(async run => { await new Promise(resolve => setTimeout(resolve, 30_000)); @@ -43,26 +56,26 @@ describe("native execution renewal ownership", () => { // Renewals start at 20s/60s and extend the database lease at 50s/90s. // Connection waits must not shorten the successfully extended ownership. await vi.advanceTimersByTimeAsync(95_000); - expect(update).toHaveBeenCalledTimes(2); + expect(returning).toHaveBeenCalledTimes(2); expect(input.onLost).not.toHaveBeenCalled(); } finally { await guard.stop().catch(() => undefined); } expect(vi.getTimerCount()).toBe(0); }); it("shares one timer across preparation and execution and stops idle SQL after both release", async () => { vi.useFakeTimers(); - const { input, update } = fixture(); + const { input, returning } = fixture(); const preparation = startNativeSessionExecutionLeaseRenewal(input); const execution = startNativeSessionExecutionLeaseRenewal(input); - expect(update).not.toHaveBeenCalled(); + expect(returning).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(40_000); - expect(update).toHaveBeenCalledTimes(2); + expect(returning).toHaveBeenCalledTimes(2); await preparation.stop(); await preparation.stop(); await vi.advanceTimersByTimeAsync(20_000); - expect(update).toHaveBeenCalledTimes(3); + expect(returning).toHaveBeenCalledTimes(3); await execution.stop(); await vi.advanceTimersByTimeAsync(120_000); - expect(update).toHaveBeenCalledTimes(3); + expect(returning).toHaveBeenCalledTimes(3); expect(vi.getTimerCount()).toBe(0); }); it("retries cancelled SQL within the original deadline without losing healthy ownership", async () => { @@ -76,6 +89,36 @@ describe("native execution renewal ownership", () => { await guard.stop(); expect(vi.getTimerCount()).toBe(0); }); + it("renews alone at once after a shared transaction fails for a reason other than a cancelled statement", async () => { + vi.useFakeTimers(); + const { input, returning, update } = fixture(); + returning.mockRejectedValueOnce(Object.assign(new Error("integer out of range"), { code: "22003" })); + const guard = startNativeSessionExecutionLeaseRenewal({ ...input, confirmedDeadline: performance.now() + 60_000 }); + await vi.advanceTimersByTimeAsync(20_000); + // The batch UPDATE failed, then the single path renewed this run alone. + expect(returning).toHaveBeenCalledTimes(2); + expect(update).toHaveBeenCalledTimes(1); + expect(guard.remainingMs()).toBe(60_000); + expect(input.onLost).not.toHaveBeenCalled(); + await guard.stop(); + }); + it("does not retry every second when the shared and the single renewal both fail (database outage)", async () => { + vi.useFakeTimers(); + const { input, returning, coordinator } = fixture(); + const outage = Object.assign(new Error("Connection terminated unexpectedly"), { code: "08006" }); + returning.mockRejectedValueOnce(outage).mockRejectedValueOnce(outage); + coordinator.mockRejectedValueOnce(outage); + const guard = startNativeSessionExecutionLeaseRenewal({ ...input, confirmedDeadline: performance.now() + 60_000 }); + await vi.advanceTimersByTimeAsync(20_000); + expect(returning).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(19_000); + expect(returning).toHaveBeenCalledTimes(2); + // The next interval goes back to the shared batch. + await vi.advanceTimersByTimeAsync(1_000); + expect(returning).toHaveBeenCalledTimes(3); + expect(guard.remainingMs()).toBe(60_000); + await guard.stop(); + }); it("detaches at the deadline while a database response is missing and cannot regain authority from a late response", async () => { vi.useFakeTimers(); const { input, returning } = fixture(); @@ -129,7 +172,9 @@ describe("native execution renewal ownership", () => { }); it("reacquires a lapsed local lease with the generation of the attempt that took over", async () => { vi.useFakeTimers(); - const { input, update, returning } = fixture(); + const { input, update, returning, lock } = fixture(); + // The local lease lapsed (host sleep): only the single-run path may take it back. + lock.live = false; const conditions: unknown[] = []; update.mockImplementation(() => ({ set: () => ({ where: (condition: unknown) => { conditions.push(condition); return { returning }; } }) })); const local = { ...input, remote: false, controllerGeneration: 3 }; @@ -187,15 +232,15 @@ describe("native execution renewal ownership", () => { }); it("does not queue SQL behind a slow renewal and fences a lost owner", async () => { vi.useFakeTimers(); - const { input, update, returning } = fixture(); + const { input, returning } = fixture(); let complete!: (value: { runId: string }[]) => void; returning.mockImplementationOnce(() => new Promise(resolve => { complete = resolve; })); const guard = startNativeSessionExecutionLeaseRenewal(input); await vi.advanceTimersByTimeAsync(50_000); - expect(update).toHaveBeenCalledTimes(1); + expect(returning).toHaveBeenCalledTimes(1); complete([]); await vi.advanceTimersByTimeAsync(100_000); - expect(update).toHaveBeenCalledTimes(1); + expect(returning).toHaveBeenCalledTimes(1); expect(input.onLost).toHaveBeenCalledTimes(1); expect(() => startNativeSessionExecutionLeaseRenewal(input)).toThrow("native_session_lease_lost"); await expect(guard.stop()).rejects.toThrow("native_session_lease_lost"); diff --git a/server/src/services/native-runtime/native-execution-lease.ts b/server/src/services/native-runtime/native-execution-lease.ts index 626fe99ed33..a46820364c3 100644 --- a/server/src/services/native-runtime/native-execution-lease.ts +++ b/server/src/services/native-runtime/native-execution-lease.ts @@ -1,4 +1,3 @@ -import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import type { Db } from "@paperclipai/db"; import { environmentLeases, heartbeatRuns, livenessDb, nativeRunFinalizations } from "@paperclipai/db"; import { and, eq, gt, isNotNull, isNull, notExists, or, sql, type SQL } from "drizzle-orm"; @@ -316,7 +315,9 @@ async function renewNativeSessionExecutionLeaseDeadline( // but conservatively include UPDATE/commit response time without assuming // that this server's wall clock agrees with the database's clock. confirmedDeadline = performance.now() + leaseTtlMs; - await notifyLiveRunRecovery(tx, { companyId: input.companyId, runId: input.runId }); + // No recovery signal: a renewal only moves the deadline later. Recovery + // keeps its deadline at the earlier expiry and re-reads the row then, so a + // lapsed lease is still found at its expiry (as for the legacy lease). return tx.update(nativeRunFinalizations) .set({ leaseExpiresAt: sql`clock_timestamp() + (${leaseTtlMs} * interval '1 millisecond')`, @@ -356,14 +357,219 @@ type RenewalInput = { }; export interface NativeExecutionLeaseRenewal { stop(): Promise; remainingMs(): number; } -// Preparation and execution share one renewal timer. Each caller releases its -// own reference; the final release drains any SQL already in flight. -const renewals = new WeakMap; +type RenewalState = { + input: RenewalInput; references: number; pending: Promise; busy: boolean; lost: Error | null; deadline: ReturnType; loss: Promise; confirmedDeadline: number; adopt(confirmedDeadline: number): void; -}>>(); + /** `performance.now()` at which the next renewal is due, and the cadence. */ + nextRenewAt: number; intervalMs: number; ttlMs: number; + /** Renews alone (not in a shared batch) after its own renewal failed where others succeeded. */ + solo: boolean; + /** Settles one renewal attempt of `binding` from its batch outcome. */ + renew(outcome: BatchOutcome, binding: RenewalInput): Promise; +}; +type RenewalResult = "renewed" | "failed" | "settled"; + +/** A cancelled statement or a deadlock says nothing about the lease; retry it. */ +function retryableRenewalError(error: unknown) { + for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { + // 40P01: taking back a lapsed lease locks the run after the coordinator; + // a writer holding them in the other order can deadlock. + const code = (cause as { code?: unknown }).code; + if (code === "57014" || code === "40P01") return true; + } + return false; +} + +// Preparation and execution share one renewal. Each caller releases its own +// reference; the final release drains any SQL already in flight. +const renewals = new WeakMap>(); + +/** + * Batched renewal (one shared schedule per database). + * + * Every lease of this process is renewed by one timer. When the first lease + * comes due, all leases due within a quarter of their own interval are renewed + * with it, so the leases of a process converge on one batch per interval. A + * batch is one transaction of constant size per `RENEWAL_BATCH_MAX_ROWS` rows + * (statement timeout, row locks, update), instead of one transaction per run. + * + * The batch keeps the single-run guarantees: + * - It locks each coordinator row before it reads the expiry, as the single + * path does, so an owner whose lease expired while another writer held the + * row is never revived (the UPDATE alone could test a stale expiry). + * - It extends only a row that this exact owner still holds: run, company, + * task, lease owner, attempt, controller boot id, pid and process start, and + * a live lease. A local lease that lapsed (host sleep) and a row that + * another writer has locked are not decided here: they renew alone through + * `renewNativeSessionExecutionLeaseDeadline`, which takes the run row lock + * for the lapsed case and waits for the busy row, exactly as before. + * - The confirmed deadline is sampled after the locks and before the UPDATE. + * - A run whose row was locked live and not extended fails with + * `native_session_lease_lost`, independently of the other runs. + * Like the single path, a batch sends no recovery signal: a renewal only moves + * a deadline later, and recovery re-reads the row at the deadline it holds. + * A batch that is cancelled (statement timeout or deadlock) extends nothing, + * and its runs retry after `RENEWAL_BATCH_RETRY_MS`. Any other batch failure, + * also one thrown while the statement is built, may come from one bad row, so + * each of its runs renews alone at once and only a bad row's own run can fail. + * A run that fails alone while others in its batch renew keeps renewing + * alone. During an outage every single renewal fails too; those runs return + * to the batch at their next interval, not after a short retry. + */ +const RENEWAL_BATCH_MAX_ROWS = 200; +const RENEWAL_BATCH_RETRY_MS = 1_000; +/** A batch still running after this long no longer holds back the next one. */ +const RENEWAL_BATCH_STALLED_MS = 15_000; + +type BatchOutcome = { renewed: number } | { lost: true } | { alone: true } | { failed: unknown }; + +type BatchRow = { state: RenewalState; binding: RenewalInput }; + +async function renewNativeExecutionLeaseBatch(db: Db, rows: BatchRow[]): Promise> { + const outcomes = new Map(); + const values = (items: BatchRow[], withOwner: boolean) => sql.join(items.map(({ binding, state }) => withOwner + ? sql`(${binding.runId}::uuid, ${binding.companyId}::uuid, ${binding.issueId}::uuid, ${binding.leaseOwner}::text, ${binding.attempt}::int, + ${binding.controller.bootId}::text, ${binding.controller.pid}::int, ${binding.controller.processStartedAt.toISOString()}::timestamptz, + ${state.ttlMs}::int)` + : sql`(${binding.runId}::uuid, ${binding.companyId}::uuid, ${binding.issueId}::uuid, ${binding.leaseOwner}::text)`), sql`, `); + await livenessDb(db).transaction(async tx => { + await tx.execute(sql`select set_config('statement_timeout', ${`${RENEWAL_STATEMENT_TIMEOUT_MS}ms`}, true)`); + // Lock first, then read the expiry from the locked row version. A row that + // another writer holds is skipped and renews alone, so one busy run cannot + // hold the whole batch until its statement timeout. + const locked = await tx.execute<{ run_id: string; live: boolean }>(sql` + with requested(run_id, company_id, issue_id, lease_owner) as (values ${values(rows, false)}) + select f.run_id::text as run_id, coalesce(f.lease_expires_at > clock_timestamp(), false) as live + from ${nativeRunFinalizations} f + join requested q on f.run_id = q.run_id and f.company_id = q.company_id + and f.issue_id = q.issue_id and f.lease_owner = q.lease_owner + order by f.run_id + for update of f skip locked`); + const lockedRows = new Map(); + for (const row of (locked as unknown as Array<{ run_id: string; live: boolean }> | undefined) ?? []) { + lockedRows.set(row.run_id, row.live); + } + const sampledAt = performance.now(); + const live: BatchRow[] = []; + for (const row of rows) { + const isLive = lockedRows.get(row.binding.runId); + // Not locked: missing, another owner's, or busy. The single path decides. + if (isLive === undefined) outcomes.set(row.state, { alone: true }); + else if (isLive) live.push(row); + // A lapsed local lease may be taken back only under the run row lock. + else outcomes.set(row.state, reacquires(row.binding) ? { alone: true } : { lost: true }); + } + if (live.length === 0) return; + const renewed = await tx.execute<{ run_id: string }>(sql` + with requested(run_id, company_id, issue_id, lease_owner, attempt, boot_id, pid, process_started_at, ttl_ms) + as (values ${values(live, true)}) + update ${nativeRunFinalizations} f + set lease_expires_at = clock_timestamp() + (q.ttl_ms * interval '1 millisecond'), updated_at = clock_timestamp() + from requested q + where f.run_id = q.run_id and f.company_id = q.company_id and f.issue_id = q.issue_id + and f.lease_owner = q.lease_owner and f.attempt = q.attempt + and f.controller_boot_id = q.boot_id and f.controller_pid = q.pid + and f.controller_process_started_at = q.process_started_at + and f.lease_expires_at > clock_timestamp() + returning f.run_id::text as run_id`); + const renewedIds = new Set(((renewed as unknown as Array<{ run_id: string }> | undefined) ?? []).map(row => row.run_id)); + for (const row of live) { + outcomes.set(row.state, renewedIds.has(row.binding.runId) ? { renewed: sampledAt + row.state.ttlMs } + // A local lease that lapsed between the lock and the update may still + // be taken back; only the single path can decide that. + : reacquires(row.binding) ? { alone: true } : { lost: true }); + } + }); + return outcomes; +} + +function reacquires(binding: RenewalInput) { + return !binding.remote && binding.controllerGeneration !== undefined; +} + +type RenewalSchedule = { leases: Set; timer?: ReturnType; flushing: number | null }; +const schedules = new WeakMap(); + +function renewalSchedule(db: Db): RenewalSchedule { + let schedule = schedules.get(db); + if (!schedule) schedules.set(db, schedule = { leases: new Set(), flushing: null }); + return schedule; +} + +function scheduleRenewals(db: Db, schedule: RenewalSchedule) { + clearTimeout(schedule.timer); + schedule.timer = undefined; + let at = Infinity; + for (const state of schedule.leases) if (!state.busy && !state.lost) at = Math.min(at, state.nextRenewAt); + if (at === Infinity) return; + if (schedule.flushing !== null) at = Math.max(at, schedule.flushing + RENEWAL_BATCH_STALLED_MS); + schedule.timer = setTimeout(() => runDueRenewals(db, schedule), Math.max(0, at - performance.now())); + schedule.timer.unref?.(); +} + +function runDueRenewals(db: Db, schedule: RenewalSchedule) { + schedule.timer = undefined; + const now = performance.now(); + if (schedule.flushing !== null && now - schedule.flushing < RENEWAL_BATCH_STALLED_MS) { + scheduleRenewals(db, schedule); + return; + } + const due = [...schedule.leases].filter(state => !state.busy && !state.lost + && state.nextRenewAt - state.intervalMs / 4 <= now); + if (!due.some(state => state.nextRenewAt <= now)) { scheduleRenewals(db, schedule); return; } + startRenewalBatches(db, schedule, due); +} + +/** + * Renews every lease of this process that is not renewing already, now, as one + * batch, and resolves when each of them has settled. For tests and diagnostics; + * the shared timer renews on its own. + */ +export async function renewNativeExecutionLeasesNow(db: Db): Promise { + const schedule = schedules.get(db); + if (!schedule) return; + const due = [...schedule.leases].filter(state => !state.busy && !state.lost); + if (due.length === 0) return; + startRenewalBatches(db, schedule, due); + await Promise.all(due.map(state => state.pending)); +} + +function startRenewalBatches(db: Db, schedule: RenewalSchedule, due: RenewalState[]) { + const now = performance.now(); + for (const state of due) { state.busy = true; state.nextRenewAt = now + state.intervalMs; } + const flush = now; + schedule.flushing = flush; + const batches: Promise[] = []; + for (const state of due) if (state.solo) state.pending = state.renew({ alone: true }, state.input).then(() => undefined); + const batched = due.filter(state => !state.solo); + for (let offset = 0; offset < batched.length; offset += RENEWAL_BATCH_MAX_ROWS) { + const rows = batched.slice(offset, offset + RENEWAL_BATCH_MAX_ROWS).map(state => ({ state, binding: state.input })); + // An async function: an error while the statement is built rejects here too. + const result = renewNativeExecutionLeaseBatch(db, rows).catch((error: unknown) => error); + // Each run settles its own outcome (and any single renewal it needs); + // stop() drains exactly that. The next batch waits only for this one. + const settled = rows.map(row => result.then(outcomes => row.state.renew(outcomes instanceof Map + ? outcomes.get(row.state) ?? { alone: true } : { failed: outcomes }, row.binding))); + rows.forEach((row, index) => { row.state.pending = settled[index]!.then(() => undefined); }); + void result.then(outcomes => { + if (outcomes instanceof Map || retryableRenewalError(outcomes)) return; + // The batch failed but some runs renewed alone: the runs that failed + // alone are the cause, so keep them out of later batches. + void Promise.all(settled).then(results => { + if (!results.includes("renewed")) return; + results.forEach((result, index) => { if (result === "failed") rows[index]!.state.solo = true; }); + }); + }); + batches.push(result); + } + void Promise.all(batches).finally(() => { + if (schedule.flushing === flush) schedule.flushing = null; + scheduleRenewals(db, schedule); + }); + scheduleRenewals(db, schedule); +} async function attemptAdvancedUnderOwner(binding: RenewalInput): Promise { const [row] = await livenessDb(binding.db).transaction(async tx => { @@ -396,10 +602,13 @@ export function startNativeSessionExecutionLeaseRenewal(input: RenewalInput): Na state.references += 1; } else { const ttlMs = input.remote ? REMOTE_CONTROLLER_LEASE_MS : NATIVE_SESSION_EXECUTION_LEASE_TTL_MS; + const intervalMs = input.remote ? REMOTE_CONTROLLER_LEASE_MS / 3 : NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS; + const schedule = renewalSchedule(input.db); const lose = (error: unknown) => { if (state!.lost) return; state!.lost = error instanceof Error ? error : new Error("native_session_lease_lost"); - clearInterval(state!.timer); + schedule.leases.delete(state!); + scheduleRenewals(input.db, schedule); clearTimeout(state!.deadline); state!.loss = Promise.resolve().then(state!.input.onLost).catch(() => undefined); }; @@ -408,36 +617,55 @@ export function startNativeSessionExecutionLeaseRenewal(input: RenewalInput): Na timer.unref?.(); return timer; }; - const renew = () => { - if (state!.lost || state!.busy) return; - state!.busy = true; - const binding = state!.input; - state!.pending = renewNativeSessionExecutionLeaseDeadline({ ...binding, leaseTtlMs: ttlMs, - reacquire: !binding.remote && binding.controllerGeneration !== undefined - ? { controllerGeneration: binding.controllerGeneration } : undefined }) - .then(confirmedDeadline => { - // A late database response cannot restore local authority after loss - // or recreate a timer after the final caller has stopped. - if (state!.references === 0) return; - state!.adopt(confirmedDeadline); - }) - .catch(async (error: unknown) => { - if (state!.input !== binding) return; - // A cancelled statement can retry at the next tick, but does not - // extend the ownership deadline. Never queue overlapping attempts. - for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { - // 40P01: taking back a lapsed lease locks the run after the - // coordinator; a writer holding them in the other order can deadlock. - const code = (cause as { code?: unknown }).code; - if (code === "57014" || code === "40P01") return; - } - const advanced = await attemptAdvancedUnderOwner(binding).catch(() => null); - if (advanced !== false || state!.input !== binding) return; - lose(error); - }).finally(() => { state!.busy = false; }); + const renewed = (confirmedDeadline: number) => { + // A late database response cannot restore local authority after loss + // or recreate a timer after the final caller has stopped. + if (state!.references === 0) return; + state!.adopt(confirmedDeadline); + }; + const failed = async (error: unknown, binding: RenewalInput) => { + if (state!.input !== binding) return; + // A cancelled statement can retry at the next tick, but does not + // extend the ownership deadline. Never queue overlapping attempts. + if (retryableRenewalError(error)) return; + const advanced = await attemptAdvancedUnderOwner(binding).catch(() => null); + if (advanced !== false || state!.input !== binding) return; + lose(error); + }; + const renew = async (outcome: BatchOutcome, binding: RenewalInput): Promise => { + let retrySoon = false, moreSql = false; + try { + if (state!.lost) return "settled"; + if ("renewed" in outcome) { renewed(outcome.renewed); return "renewed"; } + if ("lost" in outcome) { moreSql = true; await failed(new Error("native_session_lease_lost"), binding); return "failed"; } + if ("failed" in outcome && retryableRenewalError(outcome.failed)) { + // A cancelled shared statement says nothing about any one lease: + // extend nothing and retry soon. Each run's own deadline still + // detaches it if renewals keep failing. + retrySoon = true; + state!.nextRenewAt = Math.min(state!.nextRenewAt, performance.now() + RENEWAL_BATCH_RETRY_MS); + return "settled"; + } + // Renew alone: the single path decides this row, and when the batch + // failed for another reason, a bad row costs only its own run. + moreSql = true; + return await renewNativeSessionExecutionLeaseDeadline({ ...binding, leaseTtlMs: ttlMs, + reacquire: reacquires(binding) ? { controllerGeneration: binding.controllerGeneration! } : undefined }) + .then((confirmed): RenewalResult => { renewed(confirmed); state!.solo = false; return "renewed"; }, + async (error: unknown): Promise => { await failed(error, binding); return "failed"; }); + } finally { + state!.busy = false; + // A tick that passed while this attempt was in flight is skipped, as + // a fixed interval would skip it: a slow database gets no extra load. + if (!retrySoon) { + const now = performance.now(); + while (state!.nextRenewAt <= now) state!.nextRenewAt += intervalMs; + } + // The batch reschedules once for all of its runs that settled with + // it; only a run that needed more SQL reschedules on its own. + if (moreSql) scheduleRenewals(input.db, schedule); + } }; - const timer = setInterval(renew, input.remote ? REMOTE_CONTROLLER_LEASE_MS / 3 : NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS); - timer.unref?.(); const confirmedDeadline = input.confirmedDeadline ?? performance.now() + ttlMs; const adopt = (confirmed: number) => { if (state!.lost || confirmed <= state!.confirmedDeadline) return; @@ -445,9 +673,12 @@ export function startNativeSessionExecutionLeaseRenewal(input: RenewalInput): Na state!.confirmedDeadline = confirmed; state!.deadline = deadline(confirmed - performance.now()); }; - state = { input, references: 1, timer, pending: Promise.resolve(), busy: false, lost: null, - deadline: deadline(confirmedDeadline - performance.now()), confirmedDeadline, loss: Promise.resolve(), adopt }; + state = { input, references: 1, pending: Promise.resolve(), busy: false, lost: null, + deadline: deadline(confirmedDeadline - performance.now()), confirmedDeadline, loss: Promise.resolve(), adopt, + nextRenewAt: performance.now() + intervalMs, intervalMs, ttlMs, solo: false, renew }; owners.set(input.leaseOwner, state); + schedule.leases.add(state); + scheduleRenewals(input.db, schedule); } const owned = state; let stopped = false; @@ -455,7 +686,9 @@ export function startNativeSessionExecutionLeaseRenewal(input: RenewalInput): Na if (!stopped) { stopped = true; if (--owned.references === 0) { - clearInterval(owned.timer); + const schedule = renewalSchedule(input.db); + schedule.leases.delete(owned); + scheduleRenewals(input.db, schedule); clearTimeout(owned.deadline); owners!.delete(input.leaseOwner); }