diff --git a/scripts/benchmark-retained-ui-assets.ts b/scripts/benchmark-retained-ui-assets.ts new file mode 100644 index 00000000000..48ff003cb28 --- /dev/null +++ b/scripts/benchmark-retained-ui-assets.ts @@ -0,0 +1,69 @@ +/** Shared UI asset publication at startup, against storage with a fixed + * round-trip latency. Local and synthetic; never targets real storage. + * Run with pnpm --filter @paperclipai/server exec tsx ../scripts/benchmark-retained-ui-assets.ts + * Options (env): RETAINED_UI_BENCH_ASSETS (default 400), RETAINED_UI_BENCH_RTT_MS (default 25), + * RETAINED_UI_BENCH_MAX_MS: fail when the default re-publication exceeds this budget. */ +import assert from "node:assert/strict"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { Readable } from "node:stream"; +import { RETAINED_UI_ASSET_PUBLISH_CONCURRENCY, retainedUiAssets } from "../server/src/retained-ui-assets.js"; +import type { StorageProvider } from "../server/src/storage/types.js"; + +const assets = Number(process.env.RETAINED_UI_BENCH_ASSETS ?? 400); +const rttMs = Number(process.env.RETAINED_UI_BENCH_RTT_MS ?? 25); +const budgetMs = process.env.RETAINED_UI_BENCH_MAX_MS ? Number(process.env.RETAINED_UI_BENCH_MAX_MS) : null; + +function latencyStorage() { + const objects = new Map(); + let requests = 0, streamBodies = 0; + const roundTrip = () => { requests++; return new Promise(resolve => setTimeout(resolve, rttMs)); }; + const provider: StorageProvider = { + id: "s3", + putObject: async () => { throw new Error("Expected conditional publication"); }, + async putObjectConditional(input) { + await roundTrip(); + if (!Buffer.isBuffer(input.body)) streamBodies++; + if (objects.has(input.objectKey)) throw Object.assign(new Error("At least one of the pre-conditions you specified did not hold"), { name: "PreconditionFailed", $metadata: { httpStatusCode: 412 } }); + const chunks: Buffer[] = []; + if (Buffer.isBuffer(input.body)) chunks.push(input.body); + else for await (const chunk of input.body) chunks.push(Buffer.from(chunk)); + objects.set(input.objectKey, Buffer.concat(chunks)); + return { etag: '"bench"' }; + }, + async getObject(input) { + await roundTrip(); + const bytes = objects.get(input.objectKey); + if (!bytes) throw Object.assign(new Error("Object missing"), { status: 404 }); + return { stream: Readable.from([bytes]), contentLength: bytes.length, etag: '"bench"' }; + }, + async headObject(input) { await roundTrip(); return { exists: objects.has(input.objectKey) }; }, + async deleteObject(input) { objects.delete(input.objectKey); }, + }; + return { provider, stats: () => ({ requests, streamBodies }), reset: () => { requests = 0; streamBodies = 0; } }; +} + +const directory = await fs.mkdtemp(path.join(os.tmpdir(), "retained-ui-bench-")); +try { + for (let index = 0; index < assets; index++) { + await fs.writeFile(path.join(directory, `chunk-${index}.${index.toString(16)}.js`), `export const chunk${index} = ${JSON.stringify("x".repeat(2_000 + (index % 50) * 400))};\n`); + } + const results: Record[] = []; + for (const concurrency of [1, RETAINED_UI_ASSET_PUBLISH_CONCURRENCY]) { + const storage = latencyStorage(); + const retained = retainedUiAssets(storage.provider, { concurrency }); + for (const scenario of ["first_publish", "restart_republish"] as const) { + storage.reset(); + const started = performance.now(); + await retained.publish(directory); + results.push({ concurrency, scenario, ms: Math.round(performance.now() - started), ...storage.stats() }); + } + } + console.table(results); + const restart = results.find(row => row.concurrency === RETAINED_UI_ASSET_PUBLISH_CONCURRENCY && row.scenario === "restart_republish")!; + assert.equal(restart.streamBodies, 0, "publication must send replayable Buffer bodies"); + if (budgetMs !== null) assert.ok((restart.ms as number) <= budgetMs, `restart re-publication took ${restart.ms} ms, over the ${budgetMs} ms budget`); +} finally { + await fs.rm(directory, { recursive: true, force: true }); +} diff --git a/server/src/__tests__/retained-ui-assets.test.ts b/server/src/__tests__/retained-ui-assets.test.ts index 6d1c23b9d3d..182b5883e3d 100644 --- a/server/src/__tests__/retained-ui-assets.test.ts +++ b/server/src/__tests__/retained-ui-assets.test.ts @@ -20,9 +20,8 @@ function storage() { id: "s3", putObject: async () => { throw new Error("Expected conditional publication"); }, async putObjectConditional(input) { if (objects.has(input.objectKey)) throw new Error("Precondition failed"); - const chunks: Buffer[] = []; - for await (const chunk of input.body) chunks.push(Buffer.from(chunk)); - objects.set(input.objectKey, Buffer.concat(chunks)); return { etag: '"fixture-etag"' }; + if (!Buffer.isBuffer(input.body)) throw new Error("Expected a replayable Buffer body"); + objects.set(input.objectKey, Buffer.from(input.body)); return { etag: '"fixture-etag"' }; }, async getObject(input) { const bytes = objects.get(input.objectKey); @@ -78,11 +77,11 @@ it("retries an asset the storage throttled instead of failing the publication", const put = provider.putObjectConditional!.bind(provider); let refusals = 3; provider.putObjectConditional = async input => { - if (refusals > 0) { refusals--; for await (const _chunk of input.body) { /* drain as a real client would */ } throw slowDown(); } + if (refusals > 0) { refusals--; throw slowDown(); } return put(input); }; const waits: number[] = []; - await retainedUiAssets(provider, { sleep: async ms => { waits.push(ms); } }).publish(root); + await retainedUiAssets(provider, { sleep: async ms => { waits.push(ms); }, concurrency: 1 }).publish(root); expect(waits).toEqual([500, 1_000, 2_000]); expect(objects.get("ui-assets/v1/chunk-a.js")?.toString()).toBe("alpha"); expect(objects.get("ui-assets/v1/chunk-b.js")?.toString()).toBe("beta"); @@ -142,3 +141,57 @@ it("retries an existing asset whose saved bytes storage could not serve, without expect(isTransientStorageError(Object.assign(new Error("The operation was aborted"), { name: "AbortError" }))).toBe(true); expect(isTransientStorageError(Object.assign(new Error("Object missing"), { status: 404 }))).toBe(false); }); + +// Staging, 2026-11-11: a busy workspace took 85 s to bind its listener. Its +// restart re-published several hundred already-published assets one at a time +// (a refused conditional write, then a verifying read, per asset), and each +// refused stream upload logged "An error was encountered in a non-retryable +// streaming request." Publication now sends replayable Buffer bodies and keeps +// a bounded number of assets in flight. +it("re-publishes an already-published build with bounded parallel, replayable writes", async () => { + const root = await directory(), { provider } = storage(); + for (let index = 0; index < 40; index++) await fs.writeFile(path.join(root, `chunk-${index}.js`), `export const n = ${index};`); + await retainedUiAssets(provider).publish(root); + const put = provider.putObjectConditional!.bind(provider), get = provider.getObject.bind(provider); + let inFlight = 0, peak = 0, streamBodies = 0; + const latency = async (work: () => Promise) => { + inFlight++; peak = Math.max(peak, inFlight); + try { await new Promise(resolve => setTimeout(resolve, 5)); return await work(); } finally { inFlight--; } + }; + provider.putObjectConditional = input => { if (!Buffer.isBuffer(input.body)) streamBodies++; return latency(() => put(input)); }; + provider.getObject = input => latency(() => get(input)); + await retainedUiAssets(provider).publish(root); + expect(streamBodies).toBe(0); + expect(peak).toBeGreaterThan(1); + expect(peak).toBeLessThanOrEqual(8); +}); + +it("refuses a colliding build even when another asset's storage outage fails first", async () => { + const root = await directory(), { provider, objects } = storage(); + await fs.writeFile(path.join(root, "chunk-a.js"), "alpha"); + await fs.writeFile(path.join(root, "chunk-b.js"), "beta"); + objects.set("ui-assets/v1/chunk-b.js", Buffer.from("different bytes")); + const put = provider.putObjectConditional!.bind(provider), get = provider.getObject.bind(provider); + provider.putObjectConditional = async input => { + if (input.objectKey.endsWith("chunk-a.js")) throw Object.assign(new Error("Please reduce your request rate."), { name: "SlowDown", $metadata: { httpStatusCode: 503 } }); + return put(input); + }; + // The collision is found only after the outage on chunk-a has exhausted its retries. + provider.getObject = async input => { + if (input.objectKey.endsWith("chunk-b.js")) await new Promise(resolve => setTimeout(resolve, 100)); + return get(input); + }; + const retained = retainedUiAssets(provider, { sleep: async () => {}, concurrency: 2 }); + await expect(publishRetainedUiAssetsAtStartup(retained, root, () => {})).rejects.toThrow("collides"); +}); + +it("stops starting new assets after one collides, and still refuses the build", async () => { + const root = await directory(), { provider, objects } = storage(); + for (let index = 0; index < 30; index++) await fs.writeFile(path.join(root, `chunk-${String(index).padStart(2, "0")}.js`), `v${index}`); + objects.set("ui-assets/v1/chunk-00.js", Buffer.from("different bytes")); + let writes = 0; + const put = provider.putObjectConditional!.bind(provider); + provider.putObjectConditional = input => { writes++; return put(input); }; + await expect(retainedUiAssets(provider, { concurrency: 2 }).publish(root)).rejects.toThrow("collides"); + expect(writes).toBeLessThan(30); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 4eef86ba1df..3da0d854021 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -8,7 +8,7 @@ import { joinRequestRetentionService } from "../services/join-request-retention. import { approvalRetentionService } from "../services/approval-retention.js"; import { createSecretProposalsService } from "../services/secret-proposals.js"; import { logger } from "../middleware/logger.js"; -import { getStartupRecoveryState } from "../startup-recovery-state.js"; +import { getStartupRecoveryState, startupQueuedRunPassOutstanding } from "../startup-recovery-state.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; import * as idleLocalWork from "../services/idle-local-work.js"; @@ -47,6 +47,33 @@ const cloudMigrationMocks = vi.hoisted(() => ({ migrate: vi.fn(async () => null) vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudConfiguration: cloudMigrationMocks.migrate })); vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify })); vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) })); +// A volume-free start (a deployment listen URL) with its data gate and the +// cross-controller channels it opens. Off unless a test opts in. +const volumeFreeMocks = vi.hoisted(() => ({ + config: vi.fn((): string | null => null), + data: vi.fn(async (_db: unknown) => undefined), + claim: vi.fn(async (_volumeFree: boolean) => "claimed" as const), +})); +vi.mock("../services/ephemeral-server-contract.js", async importOriginal => ({ + ...await importOriginal(), + assertEphemeralServerConfig: volumeFreeMocks.config, + assertEphemeralServerData: volumeFreeMocks.data, +})); +vi.mock("../services/native-runtime/volume-free-controller-state.js", async importOriginal => ({ + ...await importOriginal(), + claimVolumeFreeControllerState: volumeFreeMocks.claim, +})); +vi.mock("../services/deployment-notifications.js", () => ({ + deploymentNotifications: vi.fn(() => ({ ready: async () => {}, markReady: vi.fn(), subscribeCompany: vi.fn(() => () => {}), stop: async () => {} })), +})); +vi.mock("../services/native-runtime/native-cancellation-delivery.js", async importOriginal => ({ + ...await importOriginal(), + registerNativeCancellationDelivery: vi.fn(() => ({ ready: Promise.resolve(), stop: async () => {} })), +})); +vi.mock("../services/native-runtime/native-runtime-response-delivery.js", async importOriginal => ({ + ...await importOriginal(), + registerNativeRuntimeResponseDelivery: vi.fn(() => ({ ready: Promise.resolve(), stop: async () => {} })), +})); // Producer transaction/notification behavior is covered by the real PostgreSQL // GitHub worker suite; this startup seam supplies a lightweight database mock. @@ -690,6 +717,7 @@ beforeEach(() => { describe("startServer feedback export wiring", () => { beforeEach(() => { vi.clearAllMocks(); + volumeFreeMocks.config.mockReturnValue(null); deliveryWorkErrors.length = 0; process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210"; process.env.PAPERCLIP_AGENT_JWT_SECRET = "0123456789abcdef0123456789abcdef"; @@ -1952,6 +1980,108 @@ describe("startServer feedback export wiring", () => { expect(fakeServer.close).not.toHaveBeenCalled(); }); + // Staging, 2026-11-11: a busy volume-free server dispatched queued runs at + // 01:18:25 and ran its data eligibility gate at 01:21:20, 4.4 minutes after + // the container started. A refusal then abandoned work it had started. + it("refuses ineligible volume-free data before the app, the listener or any queued-run dispatch", async () => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); + volumeFreeMocks.config.mockReturnValue("postgres://paperclip@db.internal:5432/paperclip"); + volumeFreeMocks.data.mockRejectedValueOnce(new Error("Volume-free server data is not eligible: agent_files_require_migration")); + + await expect(startServer()).rejects.toThrow("agent_files_require_migration"); + + expect(volumeFreeMocks.claim).toHaveBeenCalledWith(true); + expect(volumeFreeMocks.claim.mock.invocationCallOrder[0]).toBeLessThan(volumeFreeMocks.data.mock.invocationCallOrder[0]!); + expect(createAppMock).not.toHaveBeenCalled(); + expect(heartbeatServiceFactoryMock).not.toHaveBeenCalled(); + expect(fakeServer.listen).not.toHaveBeenCalled(); + expect(deliveryWorkRegisterMock).not.toHaveBeenCalled(); + }); + + // Staging, 2026-11-11: the gate's findings included run state this boot's + // own dispatch and recovery had just written. Run first, the gate sees only + // state a previous process left: no queue, run admission, recovery, or + // dispatch of this boot can write before it. + it("checks volume-free data once, before any of this boot's own startup work can write run state", async () => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); + volumeFreeMocks.config.mockReturnValue("postgres://paperclip@db.internal:5432/paperclip"); + await expect(startServer()).resolves.toBeDefined(); + await settleStartupHousekeeping(); + expect(volumeFreeMocks.data).toHaveBeenCalledOnce(); + const gate = volumeFreeMocks.data.mock.invocationCallOrder[0]!; + expect(volumeFreeMocks.claim.mock.invocationCallOrder[0]).toBeLessThan(gate); + for (const writer of [createAppMock, heartbeatServiceFactoryMock, deliveryWorkRegisterMock, fakeServer.listen, + heartbeatServiceMock.recoverNativeRunsAfterRestart, heartbeatServiceMock.registerRunDispatchWork]) { + expect(writer).toHaveBeenCalled(); + expect(Math.min(...writer.mock.invocationCallOrder)).toBeGreaterThan(gate); + } + expect(getStartupRecoveryState().phase).toBe("ready"); + }); + + it("runs every deferred recovery pass after readiness when one throws, logs it, and keeps the failed queue registered", async () => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, heartbeatSchedulerIntervalMs: 30000 })); + const failure = new Error("residual recovery store unavailable"); + heartbeatServiceMock.registerResidualRecoveryWork.mockImplementationOnce(() => ({ ready: Promise.reject(failure) })); + heartbeatServiceMock.registerActiveRunSilenceWork.mockImplementationOnce(() => { throw new Error("silence registration broke"); }); + let releaseDispatch!: () => void; + const dispatchPass = new Promise((_resolve, reject) => { releaseDispatch = () => reject(new Error("dispatch pass failed")); }); + heartbeatServiceMock.registerRunDispatchWork.mockImplementationOnce(() => ({ ready: dispatchPass, + retries: { reconciliationSnapshot: () => ({ revision: "startup-retries:1", reconciled: true }), stop: vi.fn(async () => {}) } })); + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => 1 as unknown as ReturnType); + const starting = startServer(); + try { + await starting; + expect(getStartupRecoveryState().phase).toBe("ready"); + expect(startupQueuedRunPassOutstanding()).toBe(true); + releaseDispatch(); + // The queued-run pass failed, and "Waiting for server restart" still clears. + await vi.waitFor(() => expect(startupQueuedRunPassOutstanding()).toBe(false)); + await vi.waitFor(() => expect(heartbeatServiceMock.registerStoppedInstructionCollectionWork).toHaveBeenCalledOnce()); + // Every pass after the failures still ran. + for (const later of [heartbeatServiceMock.registerIssueStateRecoveryWork, heartbeatServiceMock.registerDeferredReceiptWork, + heartbeatServiceMock.registerProviderLimitResumeWork, heartbeatServiceMock.registerScheduledRetryCancellationWork, + heartbeatServiceMock.registerNativePendingEffectWork]) expect(later).toHaveBeenCalledOnce(); + // The failed queues were registered with the coordinator, which retries them. + expect(heartbeatServiceMock.registerResidualRecoveryWork).toHaveBeenCalledOnce(); + expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledOnce(); + const failures = vi.mocked(logger.error).mock.calls.filter(([fields]) => + typeof fields === "object" && fields !== null && "startupRecoveryPass" in fields); + expect(failures.map(([fields]) => (fields as { startupRecoveryPass: string }).startupRecoveryPass)) + .toEqual(["ResidualRecovery", "ActiveRunSilence", "RunDispatch"]); + expect(fakeServer.close).not.toHaveBeenCalled(); + } finally { releaseDispatch(); await starting.catch(() => {}); timer.mockRestore(); } + }); + + // Staging, 2026-11-11: the first full queued-run dispatch pass of a busy + // workspace took 109 s, and the passes after it another 66 s, all before + // /api/ready answered. Those queues retry on their own; readiness no longer + // waits for their first pass. + it("reports ready while the first queued-run dispatch pass still runs, then finishes it and the passes after it", async () => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, heartbeatSchedulerIntervalMs: 30000 })); + let releaseDispatch!: () => void; + const dispatchPass = new Promise(resolve => { releaseDispatch = resolve; }); + heartbeatServiceMock.registerRunDispatchWork.mockImplementationOnce(() => ({ ready: dispatchPass, + retries: { reconciliationSnapshot: () => ({ revision: "startup-retries:1", reconciled: true }), stop: vi.fn(async () => {}) } })); + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => 1 as unknown as ReturnType); + const starting = startServer(); + try { + await vi.waitFor(() => expect(getStartupRecoveryState().phase).toBe("ready"), { timeout: 2_000 }); + await starting; + // Native recovery, which adopts old runs, still finished before ready. + expect(heartbeatServiceMock.recoverNativeRunsAfterRestart).toHaveBeenCalledOnce(); + // A queued turn still says it waits for this server's recovery. + expect(startupQueuedRunPassOutstanding()).toBe(true); + expect(heartbeatServiceMock.registerScheduledRetryCancellationWork).not.toHaveBeenCalled(); + // Idle sleep stays blocked while the pass runs. + expect(idleWorkSnapshot().active).toBeGreaterThan(0); + + releaseDispatch(); + await vi.waitFor(() => expect(startupQueuedRunPassOutstanding()).toBe(false)); + await vi.waitFor(() => expect(heartbeatServiceMock.registerStoppedInstructionCollectionWork).toHaveBeenCalledOnce()); + expect(heartbeatServiceMock.registerScheduledRetryCancellationWork).toHaveBeenCalledOnce(); + } finally { releaseDispatch(); await starting.catch(() => {}); timer.mockRestore(); } + }); + it("closes the bound listener when native startup recovery fails", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, diff --git a/server/src/app.ts b/server/src/app.ts index 9d1a3960523..5d845468ad4 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1110,8 +1110,12 @@ export async function createApp( ); if (process.env.PAPERCLIP_SHARED_UI_ASSETS === "true") { const retained = retainedUiAssets(createStorageProviderFromConfig(loadStorageConfig())); - await publishRetainedUiAssetsAtStartup(retained, path.join(uiDist, "assets"), error => - logger.error({ err: error }, "Shared UI assets could not be published because storage is unavailable; serving this build from local files")); + // Publish while the routes below are built; the outcome is awaited + // before the first background work starts. The promise settles to the + // failure instead of rejecting, so it is never an unhandled rejection. + app.locals.retainedUiAssetsPublication = trackIdleWork(publishRetainedUiAssetsAtStartup(retained, path.join(uiDist, "assets"), error => + logger.error({ err: error }, "Shared UI assets could not be published because storage is unavailable; serving this build from local files")) + .then(() => null, (error: unknown) => ({ error }))); staticUi.use("/assets", retained.serve); } // Serve root/index through the same runtime HTML transform as SPA routes. @@ -1251,6 +1255,11 @@ export async function createApp( app.use(staticUi); app.use(errorHandler); + // A build whose asset names collide with different published bytes refuses + // to start, so the outcome is known before any job, queue, or run starts. + const retainedUiAssetsPublication = await (app.locals.retainedUiAssetsPublication as Promise<{ error: unknown } | null> | undefined); + if (retainedUiAssetsPublication) throw retainedUiAssetsPublication.error; + jobCoordinator.start(); const deliveryWork = createDeliveryWorkCoordinator({ owner: db, diff --git a/server/src/index.ts b/server/src/index.ts index cbddbd573c3..ad3d958e0f8 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -134,7 +134,8 @@ import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js"; import { logger } from "./middleware/logger.js"; import { startEventLoopStallMonitor } from "./event-loop-stall-monitor.js"; -import { setStartupRecoveryPhase } from "./startup-recovery-state.js"; +import { setStartupQueuedRunPassPending, setStartupRecoveryPhase } from "./startup-recovery-state.js"; +import { startupTimeline } from "./startup-timeline.js"; import { StartupRefusalError, migrationRefusalError, @@ -749,6 +750,7 @@ async function startServerWithDatabaseTeardown( await Promise.all([...clients.map((client) => endDatabaseClient(client, 5)), closeLivenessDb(db, 5), closeCheckpointDb(db, 5), closeFileStorageDb(db, 5)]); }; startupDatabase.close = closeDatabaseClients; + startupTimeline.mark("database"); setSentryRunLinkResolver(createSentryRunLinkResolver(db)); await migrateCloudConfiguration(db as any); await initializeAdapterSettings(db as any); @@ -771,6 +773,18 @@ async function startServerWithDatabaseTeardown( if (restoredCloudRuntimeIdentity) config = loadConfig(); if (isCloudManagedInstance()) initializeRuntimeConfigConsumers(config); + // The volume-free data gate can refuse to start. Its inputs are the migrated + // database and the owner marker claimed above, so it runs here: before the + // app, its delivery queues and plugins start, before the listener binds, and + // before any queued run is dispatched or any old run adopted. A refused start + // then abandons no work. It used to run minutes later, after startup had + // already dispatched queued runs. No startup step it skipped could make an + // ineligible database eligible: the migrations that clear its blockers are + // maintenance commands, and the background passes startup starts were never + // awaited before it. + if (deploymentListenUrl !== null) await assertEphemeralServerData(db as any); + startupTimeline.mark("volume_free_data_gate"); + const isWarmStandby = await createCloudWarmStandby(db as any); if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) { @@ -831,6 +845,7 @@ async function startServerWithDatabaseTeardown( if (confirmationSweep.expired > 0) { logger.info(confirmationSweep, "Expired pending confirmations superseded by newer agent requests"); } + startupTimeline.mark("startup_backfills"); if (config.deploymentMode === "authenticated") { const { createBetterAuthHandler, @@ -1007,6 +1022,7 @@ async function startServerWithDatabaseTeardown( managedPluginAutoInstall, }); + startupTimeline.mark("app_created"); if (listenPort !== requestedListenPort) { logger.warn(`Requested port is busy; using next free port (requestedPort=${requestedListenPort}, selectedPort=${listenPort})`); } @@ -1097,6 +1113,7 @@ async function startServerWithDatabaseTeardown( }); }); startupListenerBound = true; + startupTimeline.mark("listening"); // A blocked event loop lapses every controller lease and freezes every // request, so record each long block with the deltas that name its class. const eventLoopStallMonitor = startEventLoopStallMonitor({ @@ -1556,6 +1573,7 @@ async function startServerWithDatabaseTeardown( : app.locals.toolGateway.cleanupExpiredSessions()).catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed")); await timeStartupRecovery("delivery_workers", () => Promise.all([budgetEnforcementWorker, accountingWorker, fastResponseWorker, ...deliveryWorkers].map(worker => worker.ready))); await Promise.all([githubEventWorker.ready, githubContinuityWorker.ready]); + startupTimeline.mark("delivery_first_passes"); if (heartbeat) { const secretProposals = createSecretProposalsService(db as any); @@ -1778,116 +1796,139 @@ async function startServerWithDatabaseTeardown( trackHeartbeatSchedulerWork(startupHeartbeatRecovery); await startupHeartbeatRecovery; } + startupTimeline.mark("native_recovery"); - if (config.heartbeatSchedulerEnabled) await heartbeat.registerResidualRecoveryWork(deliveryWork, - () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; + // The first passes of the periodic recovery queues below (residual + // recovery, the full queued-run dispatch, watchdogs, native settlement) + // scale with saved work: minutes on a busy workspace. They need not finish + // before the server serves requests correctly, so they run after readiness, + // in this order, as tracked scheduler work: + // - Every gate that can refuse a start (the volume-free data gate, native + // recovery failing closed) and the adoption of old runs (native recovery, + // hot-restart adoption) still complete before ready. + // - Each queue is the owner of its work and is already retried, woken by + // commits, and re-run on its deadline; a first pass that finishes after + // ready is the same as any later pass. + // - Until the queued-run pass finishes, a queued turn still says it waits + // for the server to finish recovery (see startup-recovery-state.ts). + // - Idle sleep stays blocked while the passes run (tracked work), and + // shutdown waits for them like any scheduler work. + const runStartupRecoveryPasses = async () => { + if (!config.heartbeatSchedulerEnabled) return; + // Each pass is isolated: one that throws is logged and the passes after + // it still run. A pass's queue stays registered and retries on its own + // schedule (retry interval, deadlines, commit wakes). + const pass = async (name: string, work: () => unknown) => { + if (heartbeatSchedulerStopped) return; + try { await work(); } + catch (err) { logger.error({ err, startupRecoveryPass: name }, "startup recovery pass failed; its queue retries on its own schedule"); } + }; + await pass("ResidualRecovery", () => heartbeat.registerResidualRecoveryWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); // Report-only ownership reconciliation (durable ownership plan, Phase 1). // Its first pass runs in the background so it never delays readiness. - if (config.heartbeatSchedulerEnabled) void registerOwnershipReconciliationWork(db as any, deliveryWork, + void registerOwnershipReconciliationWork(db as any, deliveryWork, () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped, { enabled: async () => !(await heartbeat.resolveSchedulingSuppression()).suppressed }).ready .catch(err => logger.warn({ err }, "ownership reconciliation report pass failed")); - if (config.heartbeatSchedulerEnabled) await heartbeat.registerActiveRunSilenceWork(deliveryWork, - () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; - if (config.heartbeatSchedulerEnabled) { - const work = heartbeat.registerIssueStateRecoveryWork(deliveryWork, - () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped); - idleDeadlineOwners.watchdogs = bindIdleDeadlineOwner(work.watchdogs.reconciliationSnapshot, - heartbeat.readIdleWatchdogSchedule, () => ({ coverage: { kind: "watchdogs" } })); - await work.ready; - } - if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork, - () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; + await pass("ActiveRunSilence", () => heartbeat.registerActiveRunSilenceWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); + await pass("IssueStateRecovery", async () => { + const work = heartbeat.registerIssueStateRecoveryWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped); + idleDeadlineOwners.watchdogs = bindIdleDeadlineOwner(work.watchdogs.reconciliationSnapshot, + heartbeat.readIdleWatchdogSchedule, () => ({ coverage: { kind: "watchdogs" } })); + await work.ready; + }); + await pass("DeferredReceipt", () => heartbeat.registerDeferredReceiptWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); // A connected conversation's Retry needs the chat callback createApp registered. - if (config.heartbeatSchedulerEnabled) await heartbeat.registerProviderLimitResumeWork(deliveryWork, + await pass("ProviderLimitResume", () => heartbeat.registerProviderLimitResumeWork(deliveryWork, () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped, - () => budgetResumeDelivery.prepareChatRetry).ready; - if (config.heartbeatSchedulerEnabled) { - const work = heartbeat.registerRunDispatchWork(deliveryWork, - () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped); - idleDeadlineOwners.retries = bindIdleDeadlineOwner(work.retries.reconciliationSnapshot, readIdleRetrySchedule, - snapshot => ({ coverage: { kind: "scheduled_retries", runIds: snapshot.runIds, wakeIds: snapshot.wakeIds }, issueWaits: snapshot.issueWaits })); - await work.ready; - logger.info({ processUptimeMs: Math.round(process.uptime() * 1000) }, "startup queued-run dispatch pass complete"); - } - if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; + () => budgetResumeDelivery.prepareChatRetry).ready); + await pass("RunDispatch", async () => { + const work = heartbeat.registerRunDispatchWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped); + idleDeadlineOwners.retries = bindIdleDeadlineOwner(work.retries.reconciliationSnapshot, readIdleRetrySchedule, + snapshot => ({ coverage: { kind: "scheduled_retries", runIds: snapshot.runIds, wakeIds: snapshot.wakeIds }, issueWaits: snapshot.issueWaits })); + try { await work.ready; } + // A failed first pass still ends the "waiting for server restart" + // status: the dispatch queue now owns those turns and retries them. + finally { setStartupQueuedRunPassPending(false); } + startupTimeline.mark("queued_run_pass"); + logger.info({ processUptimeMs: Math.round(process.uptime() * 1000) }, "startup queued-run dispatch pass complete"); + }); + await pass("ScheduledRetryCancellation", () => heartbeat.registerScheduledRetryCancellationWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); - if (config.heartbeatSchedulerEnabled) await heartbeat.registerCapturedInstructionCommitWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) await heartbeat.registerInstructionDirectoryCleanupWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerNativeWorkspaceFinalizationWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - unsubscribeNativeWorkspaceFinalization = worker.stop; - server.on("close", worker.stop); - - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerRetainedNativeSessionCleanupWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - unsubscribeRetainedNativeSessionCleanup = worker.stop; - server.on("close", worker.stop); - - - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerExternalConversationSettlementWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerNativeRecoveryNoticeWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerNativeEnvironmentSettlementWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerNativeRunProjectionWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - - - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerNativeResultCommitWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) await heartbeat.registerNativeStatusReassessmentWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) await heartbeat.registerNativeReviewResponseWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) { - const worker = heartbeat.registerNativeResponsePresentationWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); - unsubscribeNativeResponsePresentation = worker.stop; - server.on("close", worker.stop); - trackHeartbeatSchedulerWork(worker.ready); - } - if (config.heartbeatSchedulerEnabled) await heartbeat.registerNativePendingEffectWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) await heartbeat.registerUnsafeWorkspaceExportWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) await heartbeat.registerAutomaticCompletionReviewWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) await heartbeat.registerObsoletePolicyReviewWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) await heartbeat.registerCancelledWorkspacePreservationWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork, - () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; + await pass("CapturedInstructionCommit", () => heartbeat.registerCapturedInstructionCommitWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("InstructionDirectoryCleanup", () => heartbeat.registerInstructionDirectoryCleanupWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("NativeWorkspaceFinalization", async () => { + const worker = heartbeat.registerNativeWorkspaceFinalizationWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + unsubscribeNativeWorkspaceFinalization = worker.stop; + server.on("close", worker.stop); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("RetainedNativeSessionCleanup", async () => { + const worker = heartbeat.registerRetainedNativeSessionCleanupWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + unsubscribeRetainedNativeSessionCleanup = worker.stop; + server.on("close", worker.stop); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("ExternalConversationSettlement", async () => { + const worker = heartbeat.registerExternalConversationSettlementWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("NativeRecoveryNotice", async () => { + const worker = heartbeat.registerNativeRecoveryNoticeWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("NativeEnvironmentSettlement", async () => { + const worker = heartbeat.registerNativeEnvironmentSettlementWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("NativeRunProjection", async () => { + const worker = heartbeat.registerNativeRunProjectionWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("NativeResultCommit", async () => { + const worker = heartbeat.registerNativeResultCommitWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("NativeStatusReassessment", () => heartbeat.registerNativeStatusReassessmentWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("NativeReviewResponse", () => heartbeat.registerNativeReviewResponseWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("NativeResponsePresentation", async () => { + const worker = heartbeat.registerNativeResponsePresentationWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); + unsubscribeNativeResponsePresentation = worker.stop; + server.on("close", worker.stop); + trackHeartbeatSchedulerWork(worker.ready); + }); + await pass("NativePendingEffect", () => heartbeat.registerNativePendingEffectWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("UnsafeWorkspaceExport", () => heartbeat.registerUnsafeWorkspaceExportWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("AutomaticCompletionReview", () => heartbeat.registerAutomaticCompletionReviewWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("ObsoletePolicyReview", () => heartbeat.registerObsoletePolicyReviewWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("CancelledWorkspacePreservation", () => heartbeat.registerCancelledWorkspacePreservationWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + await pass("StoppedInstructionCollection", () => heartbeat.registerStoppedInstructionCollectionWork(deliveryWork, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready); + startupTimeline.mark("recovery_passes"); + }; const runStartupHousekeeping = async () => { const step = (name: string, work: () => Promise) => @@ -1965,9 +2006,21 @@ async function startServerWithDatabaseTeardown( snapshot => ({ issueWaits: snapshot.issueWaits })); trackHeartbeatSchedulerWork(Promise.all([scheduled.timers.ready, scheduled.monitors.ready, scheduled.routines.ready])); }; - trackHeartbeatSchedulerWork(runStartupHousekeeping().then(() => { - if (!heartbeatSchedulerStopped) startHeartbeatScheduler(); - })); + // Same order as before: recovery passes, then housekeeping, then the + // scheduler. A pass that fails is logged and the passes after it, the + // housekeeping and the scheduler still run, as after a failed housekeeping + // step. A queue whose first pass failed stays registered and retries. + setStartupQueuedRunPassPending(true); + trackHeartbeatSchedulerWork(runStartupRecoveryPasses() + .catch((err: unknown) => { logger.error({ err }, "startup recovery passes failed; their queues retry on their own schedule"); }) + .finally(() => setStartupQueuedRunPassPending(false)) + .then(() => { + startupTimeline.log("startup recovery passes complete"); + return runStartupHousekeeping(); + }) + .then(() => { + if (!heartbeatSchedulerStopped) startHeartbeatScheduler(); + })); } else { // The heartbeat scheduler is disabled, but the orphan-sandbox cleanup sweep // is still required. A failed acquire can leak a paid provider sandbox, so @@ -2018,7 +2071,6 @@ async function startServerWithDatabaseTeardown( } if (deploymentListenUrl !== null) { - await assertEphemeralServerData(db as any); const qualification = await qualifyCloudConfigurationForCurrentImage(db as any); if (qualification && !qualification.configurationPackagesEligible) { throw new Error(`Volume-free server packages are not eligible: ${qualification.blockers.map(blocker => blocker.code).join(", ")}`); @@ -2026,6 +2078,8 @@ async function startServerWithDatabaseTeardown( } markIdleStartupComplete({ scheduledBackups: config.databaseBackupEnabled }); setStartupRecoveryPhase("ready"); + startupTimeline.mark("ready"); + startupTimeline.log("startup timeline at readiness"); deploymentHints?.markReady(); logger.info(`Server startup recovery complete on ${config.host}:${listenPort}`); void systemdNotify(["--ready", `--status=Listening on ${config.host}:${listenPort}`]).then((notified) => { diff --git a/server/src/retained-ui-assets.ts b/server/src/retained-ui-assets.ts index 5536c4d46da..bd84651fa7e 100644 --- a/server/src/retained-ui-assets.ts +++ b/server/src/retained-ui-assets.ts @@ -1,5 +1,4 @@ import { createHash } from "node:crypto"; -import { createReadStream } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { pipeline } from "node:stream/promises"; @@ -59,51 +58,82 @@ export async function publishRetainedUiAssetsAtStartup( } } +/** Assets published at once: enough to hide per-request latency, small enough + * that a throttling bucket sees a steady stream rather than a burst. */ +export const RETAINED_UI_ASSET_PUBLISH_CONCURRENCY = 8; + /** Public, immutable build assets only. Publishing before readiness lets an * older tab fetch its chunks after the last old container has disappeared. * Objects have no automatic expiry: retain them at least 30 days after the * last serving deployment, including rollback, before operator reclamation. */ -export function retainedUiAssets(provider: StorageProvider, options: { sleep?: (ms: number) => Promise } = {}): { publish(directory: string): Promise; serve: RequestHandler } { +export function retainedUiAssets(provider: StorageProvider, options: { sleep?: (ms: number) => Promise; concurrency?: number } = {}): { publish(directory: string): Promise; serve: RequestHandler } { const sleep = options.sleep ?? ((ms: number) => new Promise(resolve => setTimeout(resolve, ms))); + const concurrency = Math.max(1, Math.floor(options.concurrency ?? RETAINED_UI_ASSET_PUBLISH_CONCURRENCY)); if (provider.id !== "s3" || !provider.putObjectConditional) throw new Error("Retained UI assets require conditional shared object storage"); + async function publishAsset(filename: string, objectKey: string) { + // Build assets are small: read each once into a Buffer. The storage client + // can replay a Buffer body; it cannot replay a stream, and every refused + // stream upload, including the 412 each already-published asset answers + // on a restart, logged a "non-retryable streaming request" warning. + const bytes = await fs.readFile(filename); + const expected = createHash("sha256").update(bytes).digest("hex"); + // Storage throttles a burst: retry the whole asset instead of failing startup. + for (let attempt = 0; ; attempt++) { + const signal = AbortSignal.timeout(30_000); + try { + try { + await provider.putObjectConditional!({ objectKey, body: bytes, contentLength: bytes.length, contentType: "application/octet-stream", ifNoneMatch: "*", signal }); + } catch (error) { + // An existing key or uncertain upload may be accepted only if its + // complete bytes match. Never overwrite a colliding build asset. + // A read that storage could not serve proves nothing either way: + // keep it, so the asset is tried again instead of being judged. + const existing = await provider.getObject({ objectKey, signal }) + .catch(readError => { throw isTransientStorageError(readError) ? readError : error; }); + if (await digest(existing.stream) !== expected) throw new Error("Static build asset name collides with different published bytes"); + } + return; + } catch (error) { + if (!isTransientStorageError(error) || attempt >= publishRetryDelaysMs.length) throw error; + await sleep(publishRetryDelaysMs[attempt]!); + } + } + } async function publish(directory: string) { + const assets: Array<{ filename: string; objectKey: string }> = []; async function visit(relativeDirectory: string) { for (const entry of await fs.readdir(path.join(directory, relativeDirectory), { withFileTypes: true })) { const relative = path.posix.join(relativeDirectory, entry.name); if (entry.isDirectory()) { await visit(relative); continue; } if (relative.endsWith(".map")) continue; if (!entry.isFile() || !assetPath(relative)) throw new Error("Unsupported static build asset"); - const filename = path.join(directory, relative); - const objectKey = `${prefix}${relative}`; - // A build has hundreds of assets and storage throttles a burst. The - // upload body is a stream, so the client cannot replay it: retry the - // whole asset here, with a fresh stream, instead of failing startup. - for (let attempt = 0; ; attempt++) { - const signal = AbortSignal.timeout(30_000); - const contentLength = (await fs.stat(filename)).size; - const expected = await digest(createReadStream(filename, { signal })); - const body = createReadStream(filename, { signal }); - try { - try { - await provider.putObjectConditional!({ objectKey, body, contentLength, contentType: "application/octet-stream", ifNoneMatch: "*", signal }); - } catch (error) { - // An existing key or uncertain upload may be accepted only if its - // complete bytes match. Never overwrite a colliding build asset. - // A read that storage could not serve proves nothing either way: - // keep it, so the asset is tried again instead of being judged. - const existing = await provider.getObject({ objectKey, signal }) - .catch(readError => { throw isTransientStorageError(readError) ? readError : error; }); - if (await digest(existing.stream) !== expected) throw new Error("Static build asset name collides with different published bytes"); - } - break; - } catch (error) { - if (!isTransientStorageError(error) || attempt >= publishRetryDelaysMs.length) throw error; - await sleep(publishRetryDelaysMs[attempt]!); - } finally { body.destroy(); } - } + assets.push({ filename: path.join(directory, relative), objectKey: `${prefix}${relative}` }); } } await visit(""); + // Each asset is an independent conditional write. A build has hundreds, so + // a bounded pool replaces hundreds of serial round trips with a few dozen. + // The first failure stops new assets from starting; ones in flight finish, + // and a collision among them outranks an outage. + let next = 0; + const failure: { found: boolean; error: unknown } = { found: false, error: null }; + const worker = async () => { + while (!failure.found && next < assets.length) { + const asset = assets[next++]!; + try { await publishAsset(asset.filename, asset.objectKey); } + // A collision refuses the build; an outage only lets it start on local + // files. A storage outage on one asset must never hide a collision + // found on another. + catch (error) { + if (!failure.found || (isTransientStorageError(failure.error) && !isTransientStorageError(error))) { + failure.found = true; + failure.error = error; + } + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, assets.length) }, worker)); + if (failure.found) throw failure.error; } const serve: RequestHandler = async (req, res, next) => { if (req.method !== "GET" && req.method !== "HEAD") { next(); return; } diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 1bfe7e139b9..03b6cd3e6af 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -18,7 +18,7 @@ import { executionFailureRetryCount } from "./execution-recovery-attempt.js"; import { retryIdempotentDatabaseOperation } from "../database-retry.js"; import { workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; import { readTaskDrain } from "./task-admission.js"; -import { getStartupRecoveryState } from "../startup-recovery-state.js"; +import { getStartupRecoveryState, startupQueuedRunPassOutstanding } from "../startup-recovery-state.js"; import { hasPendingRunCleanup } from "./execution-blocker.js"; import { isAgentAwaitingSetup } from "../modules/agent-lifecycle/index.js"; import { NATIVE_RESPONSE_WAKE_WAITING_REASON } from "./native-runtime/status-arbiter.js"; @@ -532,7 +532,7 @@ export function projectExecution( // Process-local, like the drain: while this server starts, recovers, or // shuts down, its full queued-run pass has not run (or no longer runs). const serverPhase = getStartupRecoveryState().phase; - if (run.status === "queued" && serverPhase !== "ready") { + if (run.status === "queued" && startupQueuedRunPassOutstanding()) { projection.cause = "server_restarting"; projection.nextAction = serverPhase === "draining" ? "The server is restarting for an update. This turn starts automatically on the replacement server." diff --git a/server/src/startup-recovery-state.ts b/server/src/startup-recovery-state.ts index 77c0a788592..4c4a63541d8 100644 --- a/server/src/startup-recovery-state.ts +++ b/server/src/startup-recovery-state.ts @@ -2,6 +2,7 @@ export type StartupRecoveryPhase = "starting" | "recovering" | "ready" | "draini let phase: StartupRecoveryPhase = "ready"; let updatedAt = new Date().toISOString(); +let queuedRunPassPending = false; export function setStartupRecoveryPhase(next: StartupRecoveryPhase): void { phase = next; @@ -15,7 +16,21 @@ export function getStartupRecoveryState(): { return { phase, updatedAt }; } +/** The server reports ready before its first full queued-run dispatch pass + * finishes; that pass scales with the backlog. Until it does, a queued turn + * still waits on this server's recovery, not on anything the user can change. */ +export function setStartupQueuedRunPassPending(pending: boolean): void { + queuedRunPassPending = pending; +} + +/** True while this process has not yet run its full queued-run pass: before + * ready, while draining, or after ready until the first pass finishes. */ +export function startupQueuedRunPassOutstanding(): boolean { + return phase !== "ready" || queuedRunPassPending; +} + export function resetStartupRecoveryStateForTests(): void { phase = "ready"; updatedAt = new Date().toISOString(); + queuedRunPassPending = false; } diff --git a/server/src/startup-timeline.ts b/server/src/startup-timeline.ts new file mode 100644 index 00000000000..777d8f67335 --- /dev/null +++ b/server/src/startup-timeline.ts @@ -0,0 +1,26 @@ +import { logger } from "./middleware/logger.js"; + +export type StartupTimelineMark = { step: string; atMs: number; durationMs: number }; + +/** Boot-to-ready milestones, measured from process start. One log line at + * readiness (and one when the background recovery passes finish) names the + * step that took the time, without an OpenTelemetry collector. */ +export function createStartupTimeline(uptimeMs: () => number = () => process.uptime() * 1000) { + const marks: StartupTimelineMark[] = []; + return { + mark(step: string) { + const atMs = Math.round(uptimeMs()); + const previous = marks.at(-1)?.atMs ?? 0; + marks.push({ step, atMs, durationMs: atMs - previous }); + }, + marks: () => marks.map(mark => ({ ...mark })), + log(message: string) { + logger.info({ + processUptimeMs: Math.round(uptimeMs()), + startupSteps: Object.fromEntries(marks.map(mark => [mark.step, mark.durationMs])), + }, message); + }, + }; +} + +export const startupTimeline = createStartupTimeline();