diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 61d67e92312..202b276eb6a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2447,6 +2447,106 @@ for the next deadline. A failed transition write is logged and does not stop the claims that already committed. Only a failure of the candidate scan itself fails the recovery pass. +### Run starts during a deploy overlap + +A Cloud deploy starts the new server while the old server still serves. Once +the new server can start runs, it takes over the start of new runs, and the old +server leaves new starts to it. The old server never stops admitting before the +new server admits, so there is always one server that starts runs. Admission +is handed over, never paused. + +- **Marker.** The new server marks that it starts runs on its own + `controller_instances` row. It sets `dispatching_at`, the time it began + starting runs, which is set once and kept. It also sets + `dispatch_lease_expires_at`, a short lease of 20 seconds. The new server + writes the marker right after its startup queued-run dispatch pass + (`queued_run_pass` in the startup timeline). At that point its queued-run + dispatch queue is registered and has scanned the whole queue once. The same + statement creates the boot's row if the row does not exist, so the new boot's + epoch is ordered after the old boot's. The new server then publishes a + `run-start-handover` hint and sweeps its queued-run dispatch, so runs the old + server left queued start at once. Another component reads "a replacement + starts runs" as a row with a higher `epoch` than the old boot where + `dispatching_at is not null`, `dispatch_lease_expires_at > now()`, and + `stopped_at` and `fenced_at` are both null. +- **Renewal.** The new server renews the lease every 5 seconds, but only while + an older boot that is neither stopped nor fenced may still serve, and for at + most 30 minutes. A lone server, or a server whose predecessor has stopped, + writes the marker once and never renews it, so an idle database can still + sleep. The marker is written only where servers overlap and hear each + other's notifications: the instance lease is on, the notification channel is + configured, and the scheduler is enabled. A self-hosted server is unchanged. +- **The old server.** Every queued-run claim reads an in-memory flag through + the existing run-admission gate, so a claim runs no query of its own. The + flag is updated in two ways: + - The `run-start-handover` notification. A server listens for it only while + a hand-over involves it, so it cannot rely on hearing the first one. + - One read before a dispatch pass when the last read is more than 2 seconds + old. The pass waits at most 1 second for that read. + + Every statement of the hand-over runs on the dedicated one-connection + controller-lease pool, in a small transaction with a 1 second statement + timeout. That covers the reads, the marker writes, and both notifications. + None of it touches the shared pool or registers tracked database work, so a + busy shared pool on either server cannot lapse the marker, delay a release, + or stop a yield or a resume. The controller-lease pool is shared only with + legacy controller-lease renewals, and never with native lease renewals: + - A hand-over statement waits behind at most one legacy renewal batch + (bounded by 10 seconds). That fits inside the 15 seconds between a marker + renewal and the marker's expiry. + - A hand-over statement delays a legacy renewal by at most 1 second. A + legacy lease lasts 60 seconds and is renewed every 10 seconds. + - At most one read, one write, and one hint are in flight at a time. + + The `run-start-handover` notification is part of the marker write's own + transaction, so it is delivered when that transaction commits. + + If marker renewals fail for longer than the lease, the old server starts runs + again once. The first renewal that succeeds afterwards announces the marker, + and the old server yields again. Each resume therefore needs at least 15 + seconds of failed renewals, so the hand-over cannot flap faster than that. + + An unanswered read never keeps a server yielding: + - A read that fails leaves the server admitting. + - A dispatch that gets no answer within 1 second admits. + - A yield always ends when the replacement's last seen lease runs out on + this server's own clock, whether or not a read is still waiting for a + connection. While it yields, the old server rechecks every + 3 seconds on the shared worker scheduler and keeps its notification + listener on. For starts it skips, it notifies the new server's queued-run + dispatch at most once a second, so the new server starts those runs without + waiting for a sweep. +- **Who yields.** A server yields only to a newer boot by epoch that has a + live marker. It never yields to an older boot, to itself, or to a row + without the marker. Rows from releases without this rule never carry the + marker, so in a mixed-version overlap nothing changes in either direction. + A server that has no row of its own does not yield. +- **Failure.** When the new server closes run admission at shutdown (a + rollback or a failed deploy), it clears its lease and sends the hint. + Shutdown waits for that clear. The clear runs after any renewal already in + flight, so a renewal cannot restore the lease, and a clear that fails is + retried before the workers stop. It + does the same while it drains, waits in standby, or cannot dispatch. If it + crashes, the lease lapses within 20 seconds. In every case the old server's + recheck resumes admission and runs its queued-run pass again. This happens + within about 3 seconds of a clear, or about 23 seconds after a crash. +- **What counts as starting a new run.** Every queued-to-running claim counts: + wake dispatch, scheduled retries promoted to queued, continuations, Retry, + and the startup fresh-run pass. These all go through + `startNextQueuedRunForAgent`. The yield happens before a claim, and a claim + stays one atomic queued-to-running update, so a run cannot be started twice + or left half-claimed. The following continue on the old server and are + handed over at SIGTERM as before: turns of runs it already started, native + session resume and reattach, finalization, lease renewal, result + application, and wakeups it accepts (these stay queued for the new server). + +A post-deploy audit should find no run claimed by the old server more than +about 2 seconds after the new server's `dispatching_at`. That window is the +age of the last read, and it applies only when the old server was not already +listening for notifications. A replacement that starts while the server +drains writes its marker when the drain ends. The gap from `dispatching_at` to the first run +that the new server starts should be seconds. + Automatic productivity reviews are retired. Run counts, missing comments, and elapsed task time do not create review tasks or impose continuation holds. Bounded continuation, provider recovery, budget limits, explicit blockers, and normal review/approval stages remain in force. Existing productivity-review tasks, comments, assignments, and dependencies remain unchanged and readable; their historical origins still identify them as recovery work for recursion suppression. ### Issue-thread interaction resolution diff --git a/packages/db/src/migrations/0471_controller_instances_dispatch_marker.sql b/packages/db/src/migrations/0471_controller_instances_dispatch_marker.sql new file mode 100644 index 00000000000..81193dee44d --- /dev/null +++ b/packages/db/src/migrations/0471_controller_instances_dispatch_marker.sql @@ -0,0 +1,2 @@ +ALTER TABLE "controller_instances" ADD COLUMN "dispatching_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "controller_instances" ADD COLUMN "dispatch_lease_expires_at" timestamp with time zone; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0466_snapshot.json b/packages/db/src/migrations/meta/0471_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0466_snapshot.json rename to packages/db/src/migrations/meta/0471_snapshot.json index 8b570142d76..2016c014343 100644 --- a/packages/db/src/migrations/meta/0466_snapshot.json +++ b/packages/db/src/migrations/meta/0471_snapshot.json @@ -1,6 +1,6 @@ { - "id": "593f81c4-2b4d-4fb4-a3e3-848b19bc9b0f", - "prevId": "38dbf2b9-da26-4c3a-b813-f37d0c19cba2", + "id": "671a422a-0b93-4155-b097-9d35a4fd2b2b", + "prevId": "f0e17e47-6217-4080-b6b6-9d83b33e9ced", "version": "7", "dialect": "postgresql", "tables": { @@ -1417,6 +1417,34 @@ "concurrently": false, "method": "btree", "with": {} + }, + "activity_log_company_handled_for_created_idx": { + "name": "activity_log_company_handled_for_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"details\" ->> 'handledForUserId')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "(\"activity_log\".\"details\" ->> 'handledForUserId') IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -16340,6 +16368,18 @@ "primaryKey": false, "notNull": false }, + "default_decision_owner_user_id": { + "name": "default_decision_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "budget_owner_user_id": { + "name": "budget_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, "require_board_approval_for_new_agents": { "name": "require_board_approval_for_new_agents", "type": "boolean", @@ -22302,6 +22342,18 @@ "type": "timestamp with time zone", "primaryKey": false, "notNull": false + }, + "dispatching_at": { + "name": "dispatching_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dispatch_lease_expires_at": { + "name": "dispatch_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false } }, "indexes": { @@ -35606,6 +35658,34 @@ "method": "btree", "with": {} }, + "heartbeat_runs_queued_created_idx": { + "name": "heartbeat_runs_queued_created_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"heartbeat_runs\".\"status\" = 'queued'", + "concurrently": true, + "method": "btree", + "with": {} + }, "heartbeat_runs_cost_accounting_pending_idx": { "name": "heartbeat_runs_cost_accounting_pending_idx", "columns": [ @@ -36235,6 +36315,12 @@ "primaryKey": false, "notNull": false }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", @@ -51034,6 +51120,18 @@ "primaryKey": false, "notNull": false }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "decision_owner_user_id": { + "name": "decision_owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, "target_date": { "name": "target_date", "type": "date", @@ -51209,6 +51307,19 @@ "onDelete": "no action", "onUpdate": "no action" }, + "projects_environment_id_environments_id_fk": { + "name": "projects_environment_id_environments_id_fk", + "tableFrom": "projects", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, "projects_git_authorization_grant_id_connection_grants_id_fk": { "name": "projects_git_authorization_grant_id_connection_grants_id_fk", "tableFrom": "projects", @@ -62977,6 +63088,12 @@ "notNull": true, "default": false }, + "time_zone": { + "name": "time_zone", + "type": "text", + "primaryKey": false, + "notNull": false + }, "created_at": { "name": "created_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index c418c97f419..e433eeb4219 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -3284,6 +3284,13 @@ "when": 1794449318551, "tag": "0470_heartbeat_runs_queued_created_index", "breakpoints": true + }, + { + "idx": 471, + "version": "7", + "when": 1794495402720, + "tag": "0471_controller_instances_dispatch_marker", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/controller_instances.ts b/packages/db/src/schema/controller_instances.ts index 60e45a626cc..0f13943d752 100644 --- a/packages/db/src/schema/controller_instances.ts +++ b/packages/db/src/schema/controller_instances.ts @@ -6,6 +6,9 @@ import { bigint, bigserial, index, integer, pgTable, text, timestamp, uniqueInde * expired, and a fenced or lease-less process stops. Stamps name the boot id, * so a record owned by a replaced container is proven gone without sharing a * host or PID space with it. + * + * During a deploy overlap the newer boot records when it began starting runs, + * and the older boot leaves new run starts to it (run-start-handover.ts). */ export const controllerInstances = pgTable( "controller_instances", @@ -20,6 +23,10 @@ export const controllerInstances = pgTable( fencedAt: timestamp("fenced_at", { withTimezone: true }), fencedByEpoch: bigint("fenced_by_epoch", { mode: "number" }), stoppedAt: timestamp("stopped_at", { withTimezone: true }), + /** When this boot began starting queued runs; null until then, and on rows written by older releases. */ + dispatchingAt: timestamp("dispatching_at", { withTimezone: true }), + /** While in the future, an older boot that is still serving leaves new run starts to this boot. */ + dispatchLeaseExpiresAt: timestamp("dispatch_lease_expires_at", { withTimezone: true }), }, (table) => ({ epochIdx: uniqueIndex("controller_instances_epoch_idx").on(table.epoch), diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts index b14740bd38e..0a50429fd64 100644 --- a/packages/db/src/work-signals.test.ts +++ b/packages/db/src/work-signals.test.ts @@ -3,7 +3,7 @@ import { drizzle } from "drizzle-orm/postgres-js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; -import { databaseWorkScopeKey, databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; +import { databaseWorkScopeKey, databaseWorkPending, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("database work signals", () => { @@ -37,6 +37,20 @@ const support = await getEmbeddedPostgresTestSupport(); } finally { unsubscribe(); } }); + it("sends an untracked commit-bound hint that leaves nothing to reconcile", async () => { + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "untracked", event => events.push(event)); + try { + await expect(db.transaction(async tx => { + await notifyDatabaseWorkOnCommit(tx, "untracked"); + throw new Error("abort"); + })).rejects.toThrow("abort"); + await db.transaction(async tx => { await notifyDatabaseWorkOnCommit(tx, "untracked"); }); + expect(databaseWorkPending(db, "untracked")).toBe(false); + expect(events).toEqual([]); + } finally { unsubscribe(); } + }); + it("reconciles a rollback without leaving a permanent pending flag", async () => { await expect(db.transaction(async tx => { await signalDatabaseWork(tx, "rollback"); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts index 46e2071dda4..baf4ca24856 100644 --- a/packages/db/src/work-signals.ts +++ b/packages/db/src/work-signals.ts @@ -92,6 +92,18 @@ export function enableDatabaseWorkNotifications(owner: object): void { if (!scope) throw new Error("Database work notifications require createDb"); scope.notify = true; } +/** + * A commit-bound hint to other server processes, sent on the transaction's own connection. Unlike + * signalDatabaseWork it registers no local intent, so a failed transaction leaves nothing to reconcile + * on another pool. For writers whose correctness never depends on this hint (best-effort wakes). + */ +export async function notifyDatabaseWorkOnCommit(connection: object, topic: string): Promise { + const context = contexts.get(connection); + if (!context?.scope.notify) return; + if (!context.transaction) throw new Error("Database work must be registered inside a transaction"); + await (connection as Connection).execute(sql`select pg_notify(${DATABASE_WORK_CHANNEL}, ${notification(context.scope, topic)})`); +} + /** A post-commit invalidation hint, for already-persisted live UI state. */ export async function publishDatabaseWorkHint(owner: object, topic: string): Promise { const scope = contexts.get(owner)?.scope; diff --git a/server/src/__tests__/heartbeat-run-start-handover.test.ts b/server/src/__tests__/heartbeat-run-start-handover.test.ts new file mode 100644 index 00000000000..8768f38ad23 --- /dev/null +++ b/server/src/__tests__/heartbeat-run-start-handover.test.ts @@ -0,0 +1,121 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, controllerInstances, createDb, heartbeatRuns } from "@paperclipai/db"; +import { DATABASE_WORK_CHANNEL, enableDatabaseWorkNotifications, receiveDatabaseWorkNotification } from "@paperclipai/db/work-signals"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { createRunStartHandover } from "../services/run-start-handover.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +/** + * Two assembled heartbeat services on one database, each with its own pool, notification listener, + * delivery coordinator and boot: the old server that is still serving and its replacement. + */ +describe.skipIf(!support.supported)("run starts during a deploy overlap (assembled heartbeat)", () => { + let database: Awaited> | undefined; + let admin!: ReturnType; + const stops: Array<() => Promise> = []; + const errors: unknown[] = []; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("heartbeat-run-start-handover-"); + admin = createDb(database.connectionString); + }, 90_000); + afterEach(async () => { + for (const stop of stops.splice(0).reverse()) await stop(); + await admin.execute(sql`truncate companies cascade`); + await admin.delete(controllerInstances); + expect(errors.splice(0)).toEqual([]); + }); + afterAll(async () => { + await admin?.$client.end({ timeout: 1 }); + await database?.cleanup(); + }); + + async function bootServer(name: string, options: { dispatches: boolean }) { + const db = createDb(database!.connectionString, { maxConnections: 4 }); + enableDatabaseWorkNotifications(db); + const listener = createDb(database!.connectionString, { maxConnections: 1 }).$client; + await listener.listen(DATABASE_WORK_CHANNEL, payload => receiveDatabaseWorkNotification(db, payload)); + // Claims this server's pool makes: the queued-to-running transition of a run. + const claims: string[] = []; + db.$client.options.debug = (_connection, query, parameters) => { + if (/^update "heartbeat_runs" set "status"/.test(query) && parameters.includes("running")) claims.push(query); + }; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, + onError: error => errors.push(error) }); + let heartbeat!: ReturnType; + const handover = createRunStartHandover({ db, coordinator, canDispatch: () => true, + onResume: () => { void heartbeat.resumeQueuedRunsAfterDrain(); }, bootId: `boot-${name}-${randomUUID()}`, + processSpace: name, pid: 1, timing: { renewMs: 200, ttlMs: 1_500, recheckMs: 150, staleAwaitMs: 100, skipHintMinMs: 50 } }); + heartbeat = heartbeatService(db, { runStartHandover: handover }); + const dispatch = options.dispatches ? heartbeat.registerRunDispatchWork(coordinator, () => true) : null; + await Promise.all([handover.ready, dispatch?.ready]); + stops.push(async () => { + await handover.stop(); + await dispatch?.stop(); + await coordinator.stop(); + await heartbeat.drainActiveRunExecutions(); + await listener.end({ timeout: 1 }); + await db.$client.end({ timeout: 1 }); + }); + return { db, heartbeat, handover, claims }; + } + + async function fixture() { + const companyId = randomUUID(); + await admin.insert(companies).values({ id: companyId, name: "Overlap", issuePrefix: `O${companyId.slice(0, 7)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "overlap-user" }); + const [agent] = await admin.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('handover-proof')"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } }, + }).returning(); + return agent.id; + } + + const runStatus = async (id: string) => + (await admin.select({ status: heartbeatRuns.status, stdout: heartbeatRuns.stdoutExcerpt }).from(heartbeatRuns) + .where(eq(heartbeatRuns.id, id)))[0]; + + it("a wake accepted by the old server stays queued there and starts on the replacement", async () => { + const agentId = await fixture(); + const old = await bootServer("old", { dispatches: true }); + await old.handover.begin(); + const replacement = await bootServer("new", { dispatches: true }); + await replacement.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + const run = await old.heartbeat.wakeup(agentId, { source: "on_demand", reason: "manual", requestedByActorType: "system" }); + if (!run) throw new Error("wake was not accepted"); + await vi.waitFor(async () => expect((await runStatus(run.id))?.status).toBe("succeeded"), { timeout: 30_000 }); + await replacement.heartbeat.drainActiveRunExecutions(); + expect((await runStatus(run.id))?.stdout).toContain("handover-proof"); + expect(old.claims).toEqual([]); + expect(replacement.claims.length).toBeGreaterThan(0); + }, 60_000); + + it("the old server starts the waiting run itself when the replacement shuts down before starting it", async () => { + const agentId = await fixture(); + const old = await bootServer("old", { dispatches: true }); + await old.handover.begin(); + // The replacement recorded its start but never dispatches (a failed deploy). + const replacement = await bootServer("new", { dispatches: false }); + await replacement.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + const run = await old.heartbeat.wakeup(agentId, { source: "on_demand", reason: "manual", requestedByActorType: "system" }); + if (!run) throw new Error("wake was not accepted"); + await new Promise(resolve => setTimeout(resolve, 500)); + expect((await runStatus(run.id))?.status).toBe("queued"); + expect(old.claims).toEqual([]); + + await replacement.handover.release(); + await vi.waitFor(async () => expect((await runStatus(run.id))?.status).toBe("succeeded"), { timeout: 30_000 }); + await old.heartbeat.drainActiveRunExecutions(); + expect(old.claims.length).toBeGreaterThan(0); + expect(replacement.claims).toEqual([]); + }, 60_000); +}); diff --git a/server/src/__tests__/run-start-handover.test.ts b/server/src/__tests__/run-start-handover.test.ts new file mode 100644 index 00000000000..56b79603a12 --- /dev/null +++ b/server/src/__tests__/run-start-handover.test.ts @@ -0,0 +1,491 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, companies, controllerInstances, controllerLeaseDb, createDb, heartbeatRuns } from "@paperclipai/db"; +import { DATABASE_WORK_CHANNEL, enableDatabaseWorkNotifications, receiveDatabaseWorkNotification, + subscribeDatabaseWork } from "@paperclipai/db/work-signals"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { readTaskDrain, startTaskDrain, stopTaskDrain } from "../services/task-admission.js"; +import { createRunStartHandover, isRunStartHandoverActive, isYieldingRunStarts, refreshRunStartYieldIfStale, + registerRunStartHandover, type RunStartHandover } from "../services/run-start-handover.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const suite = support.supported ? describe : describe.skip; + +// Short timings keep the production order of events while the test waits seconds, not minutes. +const timing = { renewMs: 200, ttlMs: 1_500, recheckMs: 150, staleAwaitMs: 100, skipHintMinMs: 50 }; + +type Server = { + name: string; + db: ReturnType; + handover: RunStartHandover; + resumed: ReturnType; + dispatchHints: number; + stop: () => Promise; +}; + +suite("run start hand-over between an old and a new server during a deploy overlap", () => { + let temporary: Awaited> | null = null; + let admin!: ReturnType; + let companyId = ""; + let agentId = ""; + const servers: Server[] = []; + const claims: Array<{ server: string; runId: string }> = []; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("paperclip-run-start-handover-"); + admin = createDb(temporary.connectionString); + companyId = randomUUID(); + agentId = randomUUID(); + await admin.insert(companies).values({ id: companyId, name: "Paperclip", issuePrefix: "RSH", requireBoardApprovalForNewAgents: false }); + await admin.insert(agents).values({ id: agentId, companyId, name: "Builder", adapterType: "codex_local", status: "idle" }); + }, 30_000); + + afterEach(async () => { + registerRunStartHandover(null); + for (const server of servers.splice(0)) await server.stop(); + claims.length = 0; + await admin.delete(heartbeatRuns); + await admin.delete(controllerInstances); + }); + + afterAll(async () => { await temporary?.cleanup(); }); + + /** A server process: its own pool, its own boot id, and the cross-server notification listener. */ + async function bootServer(name: string, options: { + canDispatch?: () => boolean; listen?: boolean; maxConnections?: number; + timing?: Partial & { writeStatementTimeoutMs?: number; readWaitMs?: number }; + } = {}): Promise { + const canDispatch = options.canDispatch ?? (() => true); + const db = createDb(temporary!.connectionString, options.maxConnections ? { maxConnections: options.maxConnections } : undefined); + enableDatabaseWorkNotifications(db); + const listener = options.listen === false ? null : createDb(temporary!.connectionString, { maxConnections: 1 }).$client; + await listener?.listen(DATABASE_WORK_CHANNEL, payload => receiveDatabaseWorkNotification(db, payload)); + const resumed = vi.fn(); + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const handover = createRunStartHandover({ db, coordinator, canDispatch, onResume: resumed, bootId: `boot-${name}-${randomUUID()}`, + processSpace: name, pid: 1, timing: { ...timing, ...options.timing } }); + await handover.ready; + const server: Server = { name, db, handover, resumed, dispatchHints: 0, stop: async () => {} }; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.queuedRunDispatch, event => { + if (event === "settled") server.dispatchHints += 1; + }); + server.stop = async () => { + unsubscribe(); + await handover.stop(); + await coordinator.stop(); + expect(errors).toEqual([]); + await listener?.end({ timeout: 1 }); + await db.$client.end({ timeout: 1 }); + }; + servers.push(server); + return server; + } + + async function queueRun(): Promise { + const id = randomUUID(); + await admin.insert(heartbeatRuns).values({ id, companyId, agentId, status: "queued", invocationSource: "on_demand" }); + return id; + } + + /** The queue's start path: refresh a stale answer, read the in-memory flag, then the atomic claim. */ + async function dispatch(server: Server, runId: string): Promise { + const refreshing = server.handover.refreshIfStale(); + if (refreshing) await refreshing; + if (server.handover.isYielding()) return false; + const [row] = await server.db.update(heartbeatRuns).set({ status: "running", startedAt: new Date() }) + .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "queued"))).returning({ id: heartbeatRuns.id }); + if (row) claims.push({ server: server.name, runId }); + return Boolean(row); + } + + async function marker(server: Server) { + const [row] = await admin.execute(sql`select dispatching_at, dispatch_lease_expires_at > now() as live + from controller_instances where process_space = ${server.name}`) as unknown as Array<{ dispatching_at: Date | null; live: boolean | null }>; + return row; + } + + it("the old server stops starting runs once the new one starts them, and the new one starts the queued run", async () => { + // The old server booted alone long ago: it recorded its own start and holds no hand-over. + const old = await bootServer("old"); + await old.handover.begin(); + expect(old.handover.isHandingOver()).toBe(false); + const before = await queueRun(); + expect(await dispatch(old, before)).toBe(true); + + // The new server boots; while it is still starting up the old one keeps starting runs. + const next = await bootServer("new"); + const duringStartup = await queueRun(); + expect(await dispatch(old, duringStartup)).toBe(true); + + // Its queued-run dispatch is running: it records that and the old server learns from the notification. + await next.handover.begin(); + expect(next.handover.isHandingOver()).toBe(true); + expect(await marker(next)).toMatchObject({ live: true }); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + const handedOver = await queueRun(); + const hintsBefore = next.dispatchHints; + expect(await dispatch(old, handedOver)).toBe(false); + // The skipped start wakes the new server's queued-run dispatch instead of waiting for a sweep. + await vi.waitFor(() => expect(next.dispatchHints).toBeGreaterThan(hintsBefore), { timeout: 10_000 }); + expect(await dispatch(next, handedOver)).toBe(true); + expect(claims.filter(claim => claim.runId === handedOver)).toEqual([{ server: "new", runId: handedOver }]); + + // The old server's SIGTERM ends the hand-over; the new server stops renewing without further writes. + await admin.update(controllerInstances).set({ stoppedAt: sql`now()` }).where(eq(controllerInstances.processSpace, "old")); + await vi.waitFor(() => expect(next.handover.isHandingOver()).toBe(false), { timeout: 10_000 }); + }); + + it("the old server learns of the hand-over on its next start even without a notification", async () => { + // The old server cannot hear this replacement: only its own dispatch path can tell it. + const old = await bootServer("old", { listen: false }); + await old.handover.begin(); + // Its last read saw no replacement; then the replacement takes over while the old server is idle. + await old.handover.refresh(); + expect(old.handover.isActive()).toBe(false); + const next = await bootServer("new"); + await next.handover.begin(); + await new Promise(resolve => setTimeout(resolve, timing.staleAwaitMs + 50)); + // The next start waits for one fresh read instead of trusting the stale one. + const run = await queueRun(); + expect(await dispatch(old, run)).toBe(false); + expect(old.handover.isActive()).toBe(true); + expect(await dispatch(next, run)).toBe(true); + }); + + it("a yield the old server cannot confirm in time does not stop it from starting runs", async () => { + // A small pool that is busy: the old server's read waits for a connection. + const old = await bootServer("old", { maxConnections: 1, timing: { readWaitMs: 200 } }); + await old.handover.begin(); + const next = await bootServer("new"); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + // The replacement dies, and the old server's only connection is busy past the wait. + // A drizzle query runs only once awaited: start it now and let it take the connection. + const busy = controllerLeaseDb(old.db).execute(sql`select pg_sleep(1.5)`).then(() => undefined); + await new Promise(resolve => setTimeout(resolve, 50)); + await next.handover.stop(); + expect(await marker(next)).toMatchObject({ live: null }); + await new Promise(resolve => setTimeout(resolve, timing.staleAwaitMs + 50)); + const run = await queueRun(); + const started = Date.now(); + expect(await dispatch(old, run)).toBe(true); + expect(Date.now() - started).toBeLessThan(5_000); + await busy; + }); + + it("a yield ends when the replacement's lease runs out, even while every read waits for a connection", async () => { + const old = await bootServer("old", { maxConnections: 1 }); + await old.handover.begin(); + // A replacement that recorded its start and then died: its lease is never renewed. + await admin.execute(sql`insert into controller_instances (boot_id, process_space, pid, lease_expires_at, dispatching_at, dispatch_lease_expires_at) + values (${`boot-crashed-${randomUUID()}`}, 'crashed', 9, now(), now(), now() + ${`${timing.ttlMs} milliseconds`}::interval)`); + await old.handover.refresh(); + expect(old.handover.isActive()).toBe(true); + + // The old server's only connection stays busy well past the lease: no read can answer. + const busy = controllerLeaseDb(old.db).execute(sql`select pg_sleep(${(timing.ttlMs + 3_000) / 1000})`).then(() => undefined); + await new Promise(resolve => setTimeout(resolve, 50)); + const resumedAt = await vi.waitFor(() => { + expect(old.resumed).toHaveBeenCalledTimes(1); + return Date.now(); + }, { timeout: timing.ttlMs + 2_000 }); + let busyDone = false; + void busy.then(() => { busyDone = true; }); + expect(busyDone).toBe(false); + expect(old.handover.isYielding()).toBe(false); + expect(resumedAt).toBeGreaterThan(0); + await busy; + }); + + it("renewals blocked past the TTL hand starts back once, then the replacement takes them again", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + const next = await bootServer("new", { timing: { writeStatementTimeoutMs: 300 } }); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + // Every renewal is cancelled while the replacement's row is held, until its lease lapses. + let unlock!: () => void; + const locked = new Promise(resolve => { unlock = resolve; }); + let held!: () => void; + const holding = new Promise(resolve => { held = resolve; }); + const lock = admin.transaction(async tx => { + await tx.execute(sql`select 1 from controller_instances where process_space = 'new' for update`); + held(); + await locked; + }); + await holding; + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 10_000 }); + expect(old.handover.isActive()).toBe(false); + expect(await dispatch(old, await queueRun())).toBe(true); + + unlock(); + await lock; + // The first renewal that succeeds again announces the marker, and the old server leaves starts to it. + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + expect(old.resumed).toHaveBeenCalledTimes(1); + const run = await queueRun(); + expect(await dispatch(old, run)).toBe(false); + expect(await dispatch(next, run)).toBe(true); + }); + + it("with both servers' shared pools blocked past the TTL, the marker is written, renewed and released on time", async () => { + const old = await bootServer("old", { maxConnections: 1 }); + await old.handover.begin(); + const next = await bootServer("new", { maxConnections: 1 }); + // Every shared-pool connection on both servers is busy for longer than the whole hand-over below. + const blockMs = timing.ttlMs * 4; + const busy = Promise.all([old.db, next.db].map(server => + server.execute(sql`select pg_sleep(${blockMs / 1000})`).then(() => undefined))); + await new Promise(resolve => setTimeout(resolve, 50)); + const blockedUntil = Date.now() + blockMs - 100; + + await next.handover.begin(); + expect(await marker(next)).toMatchObject({ live: true }); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 1_000 }); + await new Promise(resolve => setTimeout(resolve, timing.ttlMs * 1.5)); + expect(await marker(next)).toMatchObject({ live: true }); + expect(old.handover.isActive()).toBe(true); + expect(old.resumed).not.toHaveBeenCalled(); + + const started = Date.now(); + await next.handover.release(); + expect(Date.now() - started).toBeLessThan(1_000); + expect(await marker(next)).toMatchObject({ live: null }); + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 1_000 }); + // All of the above happened while both shared pools were still blocked. + expect(Date.now()).toBeLessThan(blockedUntil); + await busy; + }); + + it("an older server that outlives the lease TTL keeps leaving starts to the renewing replacement", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + const next = await bootServer("new"); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + await new Promise(resolve => setTimeout(resolve, timing.ttlMs * 2)); + expect(next.handover.isHandingOver()).toBe(true); + expect(await marker(next)).toMatchObject({ live: true }); + expect(old.resumed).not.toHaveBeenCalled(); + const run = await queueRun(); + expect(await dispatch(old, run)).toBe(false); + expect(await dispatch(next, run)).toBe(true); + }); + + it("a lease clear that fails at shutdown is retried by a later release", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + const next = await bootServer("new", { timing: { writeStatementTimeoutMs: 300 } }); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + // The clear cannot take the row while it is held, and its statement timeout cancels it. + let unlock!: () => void; + const locked = new Promise(resolve => { unlock = resolve; }); + let held!: () => void; + const holding = new Promise(resolve => { held = resolve; }); + const lock = admin.transaction(async tx => { + await tx.execute(sql`select 1 from controller_instances where process_space = 'new' for update`); + held(); + await locked; + }); + await holding; + await next.handover.release(); + unlock(); + await lock; + expect(await marker(next)).toMatchObject({ live: true }); + expect(old.resumed).not.toHaveBeenCalled(); + + await next.handover.release(); + expect(await marker(next)).toMatchObject({ live: null }); + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 10_000 }); + }); + + it("the old server starts runs again within seconds when the new server dies after taking over", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + // A replacement that recorded its start and then died: its hand-over lease is never renewed. + await admin.execute(sql`insert into controller_instances (boot_id, process_space, pid, lease_expires_at, dispatching_at, dispatch_lease_expires_at) + values (${`boot-crashed-${randomUUID()}`}, 'crashed', 9, now(), now(), now() + ${`${timing.ttlMs} milliseconds`}::interval)`); + await old.handover.refresh(); + expect(old.handover.isActive()).toBe(true); + const run = await queueRun(); + expect(await dispatch(old, run)).toBe(false); + + // Once the lease lapses the old server resumes and re-runs its queued-run pass. + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 10_000 }); + expect(old.handover.isActive()).toBe(false); + expect(await dispatch(old, run)).toBe(true); + }); + + it("the old server resumes at once when the new server shuts down (rollback or failed deploy)", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + const next = await bootServer("new"); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + await next.handover.release(); + expect(await marker(next)).toMatchObject({ live: null }); + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 10_000 }); + const run = await queueRun(); + expect(await dispatch(old, run)).toBe(true); + }); + + it("the new server releases the hand-over while it drains and takes it back when the drain ends", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + const next = await bootServer("new", { canDispatch: () => readTaskDrain(new Date()) === null }); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + try { + startTaskDrain(); + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 10_000 }); + expect(await marker(next)).toMatchObject({ live: null }); + } finally { stopTaskDrain(); } + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + }); + + it("a clear that fails when the replacement starts draining is retried well inside the lease", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + // A long lease, so the old server can only resume early through a retried clear. + const next = await bootServer("new", { canDispatch: () => readTaskDrain(new Date()) === null, + timing: { ttlMs: 8_000, writeStatementTimeoutMs: 300 } }); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + let unlock!: () => void; + const locked = new Promise(resolve => { unlock = resolve; }); + let held!: () => void; + const holding = new Promise(resolve => { held = resolve; }); + const lock = admin.transaction(async tx => { + await tx.execute(sql`select 1 from controller_instances where process_space = 'new' for update`); + held(); + await locked; + }); + await holding; + try { + startTaskDrain(); + // The first clear is cancelled by its statement timeout while the row is held. + await new Promise(resolve => setTimeout(resolve, 700)); + expect(await marker(next)).toMatchObject({ live: true }); + unlock(); + await lock; + const unlockedAt = Date.now(); + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 10_000 }); + expect(Date.now() - unlockedAt).toBeLessThan(4_000); + expect(await marker(next)).toMatchObject({ live: null }); + } finally { + unlock(); + stopTaskDrain(); + } + }); + + it("a replacement that starts up during a drain takes run starts over once the drain ends", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + const next = await bootServer("new", { canDispatch: () => readTaskDrain(new Date()) === null }); + try { + startTaskDrain(); + await next.handover.begin(); + expect(await marker(next)).toBeUndefined(); + await old.handover.refresh(); + expect(old.handover.isActive()).toBe(false); + } finally { stopTaskDrain(); } + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + expect(await marker(next)).toMatchObject({ live: true }); + }); + + it("a shutdown release during an in-flight renewal is not undone by that renewal", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + const next = await bootServer("new"); + await next.handover.begin(); + await vi.waitFor(() => expect(old.handover.isActive()).toBe(true), { timeout: 10_000 }); + + // Hold the replacement's row so its next renewal is in flight when shutdown releases the marker. + let unlock!: () => void; + const locked = new Promise(resolve => { unlock = resolve; }); + let held!: () => void; + const holding = new Promise(resolve => { held = resolve; }); + const lock = admin.transaction(async tx => { + await tx.execute(sql`select 1 from controller_instances where process_space = 'new' for update`); + held(); + await locked; + }); + await holding; + await new Promise(resolve => setTimeout(resolve, timing.renewMs * 2)); + const release = next.handover.release(); + unlock(); + await lock; + await release; + expect(await marker(next)).toMatchObject({ live: null }); + await vi.waitFor(() => expect(old.resumed).toHaveBeenCalledTimes(1), { timeout: 10_000 }); + await new Promise(resolve => setTimeout(resolve, timing.renewMs * 3)); + expect(await marker(next)).toMatchObject({ live: null }); + expect(old.handover.isActive()).toBe(false); + }); + + it("an instance from an older release, without the marker, changes nothing", async () => { + const old = await bootServer("old"); + await old.handover.begin(); + // A newer boot from a release without this code: a lease row, no dispatching_at. + await admin.insert(controllerInstances).values({ bootId: `boot-legacy-${randomUUID()}`, processSpace: "legacy", pid: 3, + leaseExpiresAt: sql`now() + interval '90 seconds'` }); + await old.handover.refresh(); + expect(old.handover.isActive()).toBe(false); + expect(await dispatch(old, await queueRun())).toBe(true); + }); + + it("a server never yields to an older instance or to itself", async () => { + const older = await bootServer("older"); + await older.handover.begin(); + const current = await bootServer("current"); + await current.handover.begin(); + // The older boot still advertises a live hand-over lease (for example, from its own overlap). + await admin.update(controllerInstances).set({ dispatchLeaseExpiresAt: sql`now() + interval '1 minute'` }); + await current.handover.refresh(); + expect(current.handover.isActive()).toBe(current.handover.isHandingOver()); + expect(current.handover.isYielding()).toBe(false); + expect(await dispatch(current, await queueRun())).toBe(true); + }); + + it("a server that has no row of its own never yields", async () => { + const next = await bootServer("new"); + await next.handover.begin(); + const unknown = await bootServer("unknown"); + await unknown.handover.refresh(); + expect(unknown.handover.isYielding()).toBe(false); + }); + + it("a lone server records its start once and does not renew or read on the start path", async () => { + const only = await bootServer("only"); + await only.handover.begin(); + expect(only.handover.isHandingOver()).toBe(false); + const first = await admin.select({ lease: controllerInstances.dispatchLeaseExpiresAt }).from(controllerInstances); + await new Promise(resolve => setTimeout(resolve, timing.renewMs * 3)); + const later = await admin.select({ lease: controllerInstances.dispatchLeaseExpiresAt }).from(controllerInstances); + expect(later).toEqual(first); + + // Once read, a start is admitted from memory: nothing to await. + await only.handover.refresh(); + expect(only.handover.refreshIfStale()).toBeNull(); + expect(await dispatch(only, await queueRun())).toBe(true); + }); + + it("a process without a hand-over (self-hosted, lease off) admits from memory with nothing to await", () => { + registerRunStartHandover(null); + expect(isYieldingRunStarts()).toBe(false); + expect(refreshRunStartYieldIfStale()).toBeNull(); + expect(isRunStartHandoverActive()).toBe(false); + }); +}); diff --git a/server/src/index.ts b/server/src/index.ts index ad3d958e0f8..28d568c59b7 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,5 +1,6 @@ import type { IdleDeadlineOwners } from "./services/idle-deadline-types.js"; -import { closeProcessRunAdmission } from "./services/heartbeat/run-admission.js"; +import { closeProcessRunAdmission, isProcessRunAdmissionClosed } from "./services/heartbeat/run-admission.js"; +import { createRunStartHandover, registerRunStartHandover } from "./services/run-start-handover.js"; import { registerOwnershipReconciliationWork } from "./services/native-runtime/native-finalization-reconciler.js"; import { bindIdleDeadlineOwner } from "./services/idle-deadline-owner.js"; import { readIdleRetrySchedule } from "./services/heartbeat/idle-retry-schedule.js"; @@ -1309,6 +1310,23 @@ async function startServerWithDatabaseTeardown( }); heartbeatSchedulerInFlight.add(tracked); }; + // Deploy overlap: once this server starts runs, an older server still serving + // leaves new run starts to it (and starts them again if this server stops). + // Only where servers overlap and hear each other's notifications. + const runStartHandover = deploymentListenUrl !== null && controllerInstanceLeaseWork && config.heartbeatSchedulerEnabled + ? createRunStartHandover({ + db: db as any, + coordinator: deliveryWork, + canDispatch: () => !heartbeatSchedulerStopped && !isProcessRunAdmissionClosed() && !isWarmStandby() + && readTaskDrain(new Date()) === null, + onResume: () => { + if (heartbeatSchedulerStopped) return; + trackHeartbeatSchedulerWork(accountingHeartbeat.resumeQueuedRunsAfterDrain().catch(err => { + logger.error({ err }, "failed to resume queued runs after the newer server stopped starting them"); + })); + }, + }) : null; + registerRunStartHandover(runStartHandover); const waitForHeartbeatSchedulerIdle = async () => { while (heartbeatSchedulerInFlight.size > 0) { await Promise.allSettled([...heartbeatSchedulerInFlight]); @@ -1856,6 +1874,13 @@ async function startServerWithDatabaseTeardown( // status: the dispatch queue now owns those turns and retries them. finally { setStartupQueuedRunPassPending(false); } startupTimeline.mark("queued_run_pass"); + // Queued-run dispatch is running here now: take new run starts over from + // an older server, then sweep the queue so runs it left start at once. + if (runStartHandover && !heartbeatSchedulerStopped) { + await runStartHandover.begin() + .catch(err => logger.warn({ err }, "could not record that this server starts runs; an older server keeps starting them")); + if (runStartHandover.isHandingOver()) work.wake(); + } logger.info({ processUptimeMs: Math.round(process.uptime() * 1000) }, "startup queued-run dispatch pass complete"); }); await pass("ScheduledRetryCancellation", () => heartbeat.registerScheduledRetryCancellationWork(deliveryWork, @@ -2148,9 +2173,20 @@ async function startServerWithDatabaseTeardown( liveEventsWebSocketServer.close(); // Finish admitted HTTP requests while their services, workers and database // are still available. Railway must switch routing before signaling A. - await closeHttpListenerForShutdown({ server, signal, timeoutMs: 30_000, log: logger, closeRunAdmission: closeProcessRunAdmission, + // An older server still serving (rollback, failed deploy) starts runs again at once: + // the release starts when run admission closes, and shutdown waits for it below. + let runStartRelease: Promise | undefined; + await closeHttpListenerForShutdown({ server, signal, timeoutMs: 30_000, log: logger, closeRunAdmission: () => { + closeProcessRunAdmission(); + runStartRelease = runStartHandover?.release(); + }, drainRequests: abort => drainHttpRequestWork(app, abort) }); const preDrain = createPreDrainShutdownBudget({ signal, log: logger }); + await preDrain.run("run_start_handover_release", async () => { + await runStartRelease; + // A failed clear is retried once more before the workers stop. + await runStartHandover?.release(); + }); await preDrain.run("systemd_notify", () => systemdNotify(["--stopping", `--status=Stopping after ${signal}`])); heartbeatSchedulerStopped = true; @@ -2285,7 +2321,11 @@ async function startServerWithDatabaseTeardown( await finalizeServerShutdown({ signal, shutdownAppServices: async () => { - try { await appShutdown?.(); } finally { await stopControllerInstanceLease(); } + try { await appShutdown?.(); } finally { + await runStartHandover?.stop().catch(() => {}); + registerRunStartHandover(null); + await stopControllerInstanceLease(); + } }, closeHttpListener: () => closeHttpListenerForShutdown({ server, signal, log: logger }), diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index cd0da16ca72..89b335df43f 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -29,6 +29,7 @@ export const DELIVERY_QUEUES = { staleIssueLocks: "stale-issue-locks", scheduledRetryDispatch: "scheduled-retry-dispatch", queuedRunDispatch: "queued-run-dispatch", + runStartHandover: "run-start-handover", statusCardEvaluation: "status-card-evaluation", runEvidenceMigration: "run-evidence-migration", providerTraceRetention: "provider-trace-retention", diff --git a/server/src/services/deployment-notifications.ts b/server/src/services/deployment-notifications.ts index a53a5d918bf..67e5e9a59f6 100644 --- a/server/src/services/deployment-notifications.ts +++ b/server/src/services/deployment-notifications.ts @@ -6,10 +6,12 @@ import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator. import { activeNativeExecutionRunIds, subscribeActiveNativeExecutions } from "./native-runtime/native-active-runs.js"; import { isIdleTaskDrainActive, subscribeTaskDrainChanges } from "./task-admission.js"; import { subscribeAllCompanyLiveEvents } from "./live-events.js"; +import { isRunStartHandoverActive, subscribeRunStartHandoverChanges } from "./run-start-handover.js"; /** Temporary overlap, not steady-state horizontal scaling. Listen while startup * and its 120-second handoff window are active, or while this process owns a - * native execution. Browser connections alone never keep PostgreSQL awake. + * native execution, or while run starts are handed over between servers. + * Browser connections alone never keep PostgreSQL awake. * Durable queues and their existing claims remain execution authority. */ export function deploymentNotifications(db: Db, coordinator: ReturnType, input: { directDatabaseUrl: string; isWarmStandby?: () => boolean; handoffMs?: number; @@ -18,7 +20,8 @@ export function deploymentNotifications(db: Db, coordinator: ReturnType["$client"] | null = null; const wanted = () => !stopped && !isIdleTaskDrainActive() - && (handoffEndsAt === null || handoffEndsAt > Date.now() || activeNativeExecutionRunIds.size > 0); + && (handoffEndsAt === null || handoffEndsAt > Date.now() || activeNativeExecutionRunIds.size > 0 + || isRunStartHandoverActive()); async function close() { const previous = client; client = null; await previous?.end({ timeout: 1 }); @@ -45,6 +48,7 @@ export function deploymentNotifications(db: Db, coordinator: ReturnType; export interface HeartbeatServiceOptions { + /** This server's run-start hand-over; defaults to the process's registered one. */ + runStartHandover?: Pick; /** Test seam before the atomic native runtime handoff. */ beforeNativeRuntimeSelection?: (runId: string) => Promise; /** Test seam immediately before the durable chat-control admission check. */ @@ -1342,7 +1345,10 @@ export function heartbeatService( isHeartbeatRunTerminalStatus, liveRunExecutions, getSchedulingSuppression, - isRunAdmissionClosed: () => shutdownInProgress || isProcessRunAdmissionClosed(), + // A newer server that already starts runs takes new run starts during a deploy overlap. + isRunAdmissionClosed: () => shutdownInProgress || isProcessRunAdmissionClosed() + || (options.runStartHandover ? options.runStartHandover.isYielding() : isYieldingRunStarts()), + refreshRunAdmission: () => options.runStartHandover ? options.runStartHandover.refreshIfStale() : refreshRunStartYieldIfStale(), resumeExecutionWaitComments, getWorktreeExecutionCutoff, resumeQueuedCommentInterrupt, diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 8378eea4404..d51f66b3bd4 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -3,6 +3,7 @@ import { connectionIntentService } from "../connection-intents.js"; import { verifyBlockerStallWake } from "../recovery/blocker-stall-wakeups.js"; import { CONTEXT_COMPACTION_WAKE_REASON, ContextCompactionError, isContextCompactionRun, readContextCompactionRequest } from "../context-compaction.js"; import { isProcessRunAdmissionClosed } from "./run-admission.js"; +import { refreshRunStartYieldIfStale } from "../run-start-handover.js"; import { deliveredQueuedCommentIds } from "../queued-comment-delivery.js"; import { hasAutomaticInteractionDeliveryHold, INTERACTION_DELIVERY_DENIED, recheckQueuedRunInteractions, withInteractionDeliveryOrigin } from "../agent-interaction-delivery.js"; import { deferredWorkIntentWait } from "../work-intent-outbox.js"; @@ -237,6 +238,8 @@ export interface HeartbeatQueueDependencies extends Pick(run: T | null, live: { has(id: string): boolean }) => T | null; getSchedulingSuppression: () => Promise<{ suppressed: boolean; reason: "worktree_instance" | "database_restore_in_progress" | "task_drain" | null }>; isRunAdmissionClosed?: () => boolean; + /** Before a dispatch pass, refresh admission inputs that have gone stale; null when current. */ + refreshRunAdmission?: () => Promise | null; canStartNewExternalBotWork: (agent: typeof agents.$inferSelect) => Promise; executeRun: (runId: string) => Promise; releaseIssueExecutionAndPromote: (run: Pick, options?: { suppressImmediateRecovery?: boolean; resumeStrandedQueue?: boolean; deferredPostCommitEffects?: WakeQueuePostCommitEffect[] }) => Promise; @@ -435,6 +438,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende liveRunExecutions, getSchedulingSuppression, isRunAdmissionClosed = isProcessRunAdmissionClosed, + refreshRunAdmission = refreshRunStartYieldIfStale, resumeExecutionWaitComments, getWorktreeExecutionCutoff, resumeQueuedCommentInterrupt, @@ -2091,6 +2095,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // Release admission promptly without first scanning unrelated recovery work. // Every run still passes the ordinary per-agent claim and dispatch gates. async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) { + const refreshing = refreshRunAdmission(); + if (refreshing) await refreshing; if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return; const cutoff = await getWorktreeExecutionCutoff(); const queuedRuns = await db @@ -2191,6 +2197,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } async function startNextQueuedRunForAgent(agentId: string, attachSetupRequest = false) { + // Null unless this server has not read the run-start hand-over recently. + const refreshing = refreshRunAdmission(); + if (refreshing) await refreshing; if (isRunAdmissionClosed() || (await getSchedulingSuppression()).suppressed) return []; const cutoff = await getWorktreeExecutionCutoff(); // Cancelled after the start lock is released: cancelRunInternal promotes the diff --git a/server/src/services/run-start-handover.ts b/server/src/services/run-start-handover.ts new file mode 100644 index 00000000000..1fef8c812ac --- /dev/null +++ b/server/src/services/run-start-handover.ts @@ -0,0 +1,398 @@ +import { sql, type SQL } from "drizzle-orm"; +import { controllerLeaseDb, type Db } from "@paperclipai/db"; +import { notifyDatabaseWorkOnCommit } from "@paperclipai/db/work-signals"; +import { logger } from "../middleware/logger.js"; +import { currentControllerBootId, currentProcessSpaceHash } from "./controller-process-identity.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { subscribeTaskDrainChanges, subscribeTaskDrainExpiry } from "./task-admission.js"; + +/** + * Hand-over of new run starts during a deploy overlap. + * + * A replacement server reports that it starts runs by setting `dispatching_at` + * and a short `dispatch_lease_expires_at` on its own `controller_instances` row, + * once its queued-run dispatch is running. An older server that is still + * serving then leaves NEW run starts (queued-run claims) to it; runs it already + * started carry on. The lease is renewed only while an older server may still + * be serving, so a lone server writes once and the database can still sleep. + * If the replacement stops, drains or dies, its lease is cleared or lapses and + * the older server starts runs again within seconds: the hand-over never leaves + * the company without a server that starts runs. + * + * Readers outside this module (for example a deployer) can read "a replacement + * is starting runs" as a row with a higher epoch than the old boot where + * `dispatching_at is not null and dispatch_lease_expires_at > now() + * and stopped_at is null and fenced_at is null`. + * + * Database use: every statement of this module (marker writes, reads, and the + * commit-bound notifications) runs in a small transaction on the dedicated + * one-connection controller-lease pool with a 1 s statement timeout. It never + * touches the shared pool and never registers tracked database work, so the + * worker coordinator has nothing of it to reconcile there. That pool is shared + * only with legacy controller-lease renewals: a statement here waits at most one + * legacy batch (10 s statement timeout), inside the 15 s between a renewal and + * the lease's expiry, and delays a legacy renewal (60 s lease, renewed every + * 10 s) by at most 1 s. At most one read, one write and one hint are in flight. + * An unanswered read never keeps this server yielding: a dispatch that cannot + * get an answer within a second admits, and a yield always ends when the + * replacement's last seen lease runs out on this server's clock. + */ +export const RUN_START_HANDOVER_RENEW_MS = 5_000; +export const RUN_START_HANDOVER_TTL_MS = 20_000; +/** A replacement renews the hand-over for at most this long after it began starting runs. */ +export const RUN_START_HANDOVER_MAX_MS = 30 * 60_000; +/** A yielding server checks this often whether the replacement still starts runs. */ +export const RUN_START_YIELD_RECHECK_MS = 3_000; + +const DEFAULT_TIMING = { + renewMs: RUN_START_HANDOVER_RENEW_MS, + ttlMs: RUN_START_HANDOVER_TTL_MS, + maxMs: RUN_START_HANDOVER_MAX_MS, + /** An older boot counts as possibly serving while its lease was live this recently. */ + olderBootWindowMs: RUN_START_HANDOVER_MAX_MS, + recheckMs: RUN_START_YIELD_RECHECK_MS, + /** A dispatch pass first waits for a read when the last one is older than this: the bound on a stale answer. */ + staleAwaitMs: 2_000, + /** A dispatch waits at most this long for that read; the read itself is cancelled by the database. */ + readWaitMs: 1_000, + readStatementTimeoutMs: 1_000, + /** Short, so a marker write can delay a legacy lease renewal on the shared lease connection by at most this. */ + writeStatementTimeoutMs: 1_000, + /** Skipped starts are announced to the replacement at most this often. */ + skipHintMinMs: 1_000, + /** A clear that failed while this server cannot start runs is retried this often until the lease would lapse. */ + clearRetryMs: 1_000, +}; + +const interval = (ms: number) => sql`${`${ms} milliseconds`}::interval`; + +export type RunStartHandover = ReturnType; + +export function createRunStartHandover(input: { + db: Db; + /** The server's shared worker coordinator: renewals and rechecks are its deadlines. */ + coordinator: ReturnType; + /** This server may start runs now (not stopping, draining, or in standby). */ + canDispatch: () => boolean; + /** Called when a yielding server starts runs again. */ + onResume?: () => void; + bootId?: string; + processSpace?: string; + pid?: number; + timing?: Partial; +}) { + const { db } = input; + const t = { ...DEFAULT_TIMING, ...input.timing }; + const bootId = input.bootId ?? currentControllerBootId(); + let stopped = false; + const listeners = new Set<() => void>(); + const changed = () => { for (const listener of listeners) { try { listener(); } catch { /* never blocks */ } } }; + + // Replacement side. Every marker write runs in this one chain, so a release + // always lands after a renewal that was already in flight. + let intendedAt: number | null = null; + let released = false; + let olderMayServe = true; + let leaseHeld = false; + /** A marker was written and no clear has succeeded since: a release must (re)try the clear. */ + let markerMayBeLive = false; + let lastWriteAt = Number.NEGATIVE_INFINITY; + let lastClearAttemptAt = Number.NEGATIVE_INFINITY; + let writes: Promise = Promise.resolve(); + const handingOver = () => intendedAt !== null && !released && olderMayServe && Date.now() - intendedAt < t.maxMs; + let reportedHandingOver = false; + const reportHandingOver = () => { + const now = handingOver(); + if (now !== reportedHandingOver) { reportedHandingOver = now; changed(); } + }; + + // Older-server side: the in-memory flag every run start reads. + let yieldingTo: string | null = null; + /** Local-clock end of the replacement's last seen lease: a yield is never kept past it, read or no read. */ + let yieldUntil = Number.NEGATIVE_INFINITY; + let lastReadAt = Number.NEGATIVE_INFINITY; + let reading: Promise | null = null; + let readRequested = false; + let lastSkipHintAt = Number.NEGATIVE_INFINITY; + let hinting: Promise | null = null; + + /** + * One statement under a statement timeout, so a slow database cancels it instead of holding a pool connection. + * With `announce`, the marker-change notification is part of the same transaction on the same connection: it is + * delivered on commit, and nothing in the write path waits on another pool. + */ + function bounded(statement: SQL, timeoutMs: number, announce = false): Promise { + // Every statement of this module runs here: never on the shared pool, whose stalls must not move run starts. + return controllerLeaseDb(db).transaction(async tx => { + await tx.execute(sql`select set_config('statement_timeout', ${String(timeoutMs)}, true)`); + // Commit-bound and untracked: a failed write leaves no uncertain transaction for the coordinator to + // reconcile on the shared pool before the next renewal. + if (announce) await notifyDatabaseWorkOnCommit(tx, DELIVERY_QUEUES.runStartHandover); + return await tx.execute(statement) as unknown as T[]; + }); + } + + const serialize = (work: () => Promise) => { + const next = writes.then(work); + writes = next.catch(error => { logger.warn({ err: error }, "run start hand-over write failed; its lease lapses unless a later write succeeds"); }); + return writes; + }; + + async function writeLease(announce: boolean) { + const [row] = await bounded<{ olderMayServe: boolean }>(sql` + insert into controller_instances (boot_id, process_space, pid, lease_expires_at, dispatching_at, dispatch_lease_expires_at) + values (${bootId}, ${input.processSpace ?? currentProcessSpaceHash()}, ${input.pid ?? process.pid}, now(), now(), + now() + ${interval(t.ttlMs)}) + on conflict (boot_id) do update set + dispatching_at = coalesce(controller_instances.dispatching_at, now()), + dispatch_lease_expires_at = now() + ${interval(t.ttlMs)} + where controller_instances.fenced_at is null and controller_instances.stopped_at is null + returning exists (select 1 from controller_instances older + where older.epoch < controller_instances.epoch and older.boot_id <> ${bootId} + and older.stopped_at is null and older.fenced_at is null + and older.lease_expires_at > now() - ${interval(t.olderBootWindowMs)}) as "olderMayServe"`, t.writeStatementTimeoutMs, + announce); + return row ?? null; + } + + async function clearLease() { + await bounded(sql`update controller_instances set dispatch_lease_expires_at = null + where boot_id = ${bootId} and dispatch_lease_expires_at is not null`, t.writeStatementTimeoutMs, true); + // Only a clear that succeeded ends the marker; a failed one is retried by the next release or renewal. + leaseHeld = false; + markerMayBeLive = false; + } + + /** Set, renew, or clear the marker to match whether this server can start runs now. */ + function maintainLease(): Promise { + return serialize(async () => { + if (intendedAt === null || released || !handingOver() && leaseHeld) return; + if (!input.canDispatch()) { + // Draining, standby or admission closed: the older server takes run starts back now. + if (markerMayBeLive) { + lastClearAttemptAt = Date.now(); + await clearLease(); + } + return; + } + if (Date.now() - lastWriteAt < t.renewMs / 2) return; + // Counted from the attempt, so a failing database is retried on the renewal cadence, not in a loop. + lastWriteAt = Date.now(); + markerMayBeLive = true; + let row: Awaited>; + // A first or regained marker tells the older server in the same transaction. + try { row = await writeLease(!leaseHeld); } + catch (error) { + // Unconfirmed: the next successful write counts as regained and tells the older server again. + leaseHeld = false; + throw error; + } + const regained = row !== null && !leaseHeld; + leaseHeld = row !== null; + // No older server left to hand over from: the lease lapses without further writes. + olderMayServe = row?.olderMayServe ?? false; + if (regained) { + if (olderMayServe) logger.info("this server now starts runs; an older server still serving leaves new run starts to it"); + } + }).finally(reportHandingOver); + } + + /** Read whether a newer boot starts runs now. A failed or cancelled read resumes run starts here. */ + async function read(): Promise { + let next: string | null = null; + let until = Number.NEGATIVE_INFINITY; + const askedAt = Date.now(); + try { + const [row] = await bounded<{ bootId: string; remainingMs: number }>(sql` + select newer.boot_id as "bootId", + extract(epoch from newer.dispatch_lease_expires_at - now()) * 1000 as "remainingMs" + from controller_instances me + join controller_instances newer on newer.epoch > me.epoch and newer.boot_id <> me.boot_id + where me.boot_id = ${bootId} and newer.dispatching_at is not null + and newer.dispatch_lease_expires_at > now() + and newer.stopped_at is null and newer.fenced_at is null + order by newer.epoch desc limit 1`, t.readStatementTimeoutMs); + next = row?.bootId ?? null; + // Counted from when the read was asked, so time spent waiting for the answer only shortens the yield. + if (row) until = askedAt + Number(row.remainingMs); + } catch (error) { + logger.warn({ err: error }, "run start hand-over read failed; this server keeps starting runs"); + } + lastReadAt = Date.now(); + if (stopped) return; + const previous = yieldingTo; + yieldingTo = next && until > Date.now() ? next : null; + yieldUntil = until; + next = yieldingTo; + if (next && !previous) { + logger.info({ replacementBootId: next }, "a newer server is starting runs; this server leaves new run starts to it"); + skipHint(true); + changed(); + worker.wake(); + } else if (!next && previous) { + logger.info({ replacementBootId: previous }, "the newer server stopped starting runs; this server starts runs again"); + changed(); + input.onResume?.(); + } + // Schedule the next recheck from this answer. + if (yieldingTo) worker.wake(); + } + + /** The replacement's lease ran out on this server's clock: start runs again even if no read has answered. */ + function expireYield() { + if (!yieldingTo || Date.now() < yieldUntil) return; + logger.warn({ replacementBootId: yieldingTo }, "the newer server's run start lease ran out; this server starts runs again"); + yieldingTo = null; + changed(); + queueMicrotask(() => input.onResume?.()); + } + + /** Single flight: a slow read stays tracked until it settles, so reads never pile up on the pool. */ + function refresh(): Promise { + reading ??= read().finally(() => { reading = null; }); + return reading; + } + + /** Runs this server skips stay queued; wake the replacement's dispatch so they do not wait for a sweep. */ + function skipHint(force = false) { + const now = Date.now(); + // Best effort and single flight: a busy pool delays one hint, never a queue of them (the replacement's own + // queued-run redrive still covers a lost hint). + if (hinting || (!force && now - lastSkipHintAt < t.skipHintMinMs)) return; + lastSkipHintAt = now; + hinting = controllerLeaseDb(db).transaction(async tx => { + await tx.execute(sql`select set_config('statement_timeout', ${String(t.writeStatementTimeoutMs)}, true)`); + await notifyDatabaseWorkOnCommit(tx, DELIVERY_QUEUES.queuedRunDispatch); + }) + .catch(error => logger.warn({ err: error }, "queued run dispatch hint for the newer server failed")) + .finally(() => { hinting = null; }); + } + + function nextDeadline(): number | null { + const deadlines: number[] = []; + // A clear that has not succeeded yet, while this server cannot start runs: retry until the lease would lapse. + if (handingOver() && !input.canDispatch() && markerMayBeLive && Date.now() < lastWriteAt + t.ttlMs) { + deadlines.push(lastClearAttemptAt + t.clearRetryMs); + } + // Renew while handing over and able to start runs; a drain change wakes the worker otherwise. + if (handingOver() && input.canDispatch()) deadlines.push(lastWriteAt + t.renewMs); + if (yieldingTo) deadlines.push(reading ? yieldUntil : Math.min(lastReadAt + t.recheckMs, yieldUntil)); + return deadlines.length ? Math.max(Date.now(), Math.min(...deadlines)) : null; + } + + const worker = input.coordinator.register(DELIVERY_QUEUES.runStartHandover, { + retryMs: t.renewMs, + // Clearing the marker while this server drains is what lets the older one resume. + runDuringDrain: true, + // Another server's marker changed. + onSettled: () => { readRequested = true; }, + run: async () => { + if (stopped) return; + expireYield(); + if (handingOver()) await maintainLease(); + if (readRequested || (yieldingTo && !reading && Date.now() - lastReadAt >= t.recheckMs)) { + readRequested = false; + // Not awaited: a read stuck behind the pool must not hold the worker, whose lease-expiry deadline still fires. + void refresh(); + } + }, + hasPending: async () => false, + nextRunAt: nextDeadline, + }); + const stopDrainChanges = subscribeTaskDrainChanges(worker.wake); + const stopDrainExpiry = subscribeTaskDrainExpiry(worker.wake); + + return { + ready: worker.ready, + /** Replacement side: this boot starts runs now. Older boots leave new starts to it. A drain defers it. */ + async begin(): Promise { + if (stopped || released || intendedAt !== null) return; + intendedAt = Date.now(); + await maintainLease(); + worker.wake(); + }, + /** Replacement side, final (shutdown): stop claiming run starts. Older boots resume at once. */ + async release(): Promise { + released = true; + reportHandingOver(); + // Runs after any write already in flight, so nothing restores the marker afterwards. + await serialize(async () => { if (markerMayBeLive) await clearLease(); }); + }, + /** The hot-path flag: a newer server starts runs, so this one starts none. Never queries inline. */ + isYielding(): boolean { + expireYield(); + if (yieldingTo) skipHint(); + return yieldingTo !== null; + }, + /** Before a dispatch pass: when the last read is older than staleAwaitMs, wait (briefly) for a fresh one. */ + refreshIfStale(): Promise | null { + if (stopped || Date.now() - lastReadAt < t.staleAwaitMs) return null; + let timer: ReturnType | undefined; + const read = refresh(); + // The read keeps running (and stays the single flight) if this caller stops waiting. Until it + // answers, an unconfirmed yield is not trusted: this server admits rather than risk a pause. + const waited = new Promise(resolve => { + timer = setTimeout(() => { + if (reading === read && yieldingTo) { + logger.warn({ replacementBootId: yieldingTo }, "run start hand-over read is slow; this server starts runs until it answers"); + yieldingTo = null; + changed(); + } + resolve(); + }, t.readWaitMs); + }); + return Promise.race([read, waited]).finally(() => clearTimeout(timer)); + }, + refresh, + /** This server holds a hand-over an older server may be reading. */ + isHandingOver: () => handingOver(), + /** A hand-over involves this server (either side). Pure read, no refresh. */ + isActive: () => handingOver() || yieldingTo !== null, + subscribe(listener: () => void) { + listeners.add(listener); + return () => { listeners.delete(listener); }; + }, + async stop(): Promise { + await this.release(); + stopped = true; + stopDrainChanges(); + stopDrainExpiry(); + yieldingTo = null; + await worker.stop(); + // A pending read or best-effort hint is not awaited: on the shared pool it could wait behind slow work, + // and it settles (or fails) when the pool answers or closes. Neither can change the marker. + }, + }; +} + +// The process's hand-over, when this deployment can hand over (cloud overlap). +let current: RunStartHandover | null = null; +const processListeners = new Set<() => void>(); +let unsubscribeCurrent: () => void = () => {}; + +export function registerRunStartHandover(handover: RunStartHandover | null): void { + unsubscribeCurrent(); + current = handover; + unsubscribeCurrent = handover?.subscribe(() => { for (const listener of processListeners) listener(); }) ?? (() => {}); +} + +/** True while a newer server starts this server's new runs. No query on this path. */ +export function isYieldingRunStarts(): boolean { + return current?.isYielding() ?? false; +} + +/** Null (nothing to await) unless this server has not read the hand-over recently. */ +export function refreshRunStartYieldIfStale(): Promise | null { + return current?.refreshIfStale() ?? null; +} + +/** Either side of a hand-over is in progress here; a pure read for listeners. */ +export function isRunStartHandoverActive(): boolean { + return current?.isActive() ?? false; +} + +export function subscribeRunStartHandoverChanges(listener: () => void): () => void { + processListeners.add(listener); + return () => { processListeners.delete(listener); }; +}