diff --git a/server/src/__tests__/retained-ui-assets.test.ts b/server/src/__tests__/retained-ui-assets.test.ts index 221359f27a0..6d1c23b9d3d 100644 --- a/server/src/__tests__/retained-ui-assets.test.ts +++ b/server/src/__tests__/retained-ui-assets.test.ts @@ -5,7 +5,7 @@ import os from "node:os"; import path from "node:path"; import { Readable } from "node:stream"; import { afterEach, expect, it } from "vitest"; -import { retainedUiAssets } from "../retained-ui-assets.js"; +import { isTransientStorageError, publishRetainedUiAssetsAtStartup, retainedUiAssets } from "../retained-ui-assets.js"; import type { StorageProvider } from "../storage/types.js"; import { staticUiFiles } from "../middleware/static-ui.js"; import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "../middleware/idle-admission.js"; @@ -66,3 +66,79 @@ it("refuses a build which reuses a published name for different bytes", async () await expect(retained.publish(root)).rejects.toThrow("collides"); expect(objects.get("ui-assets/v1/chunk-hash.js")?.toString()).toBe("original"); }); + +// Staging, 2026-11-11: the bucket answered "SlowDown" (HTTP 503) while a new +// release published its assets. The stream body could not be replayed, the +// error escaped, and every start of the server exited within a minute. +it("retries an asset the storage throttled instead of failing the publication", async () => { + const root = await directory(), { provider, objects } = storage(); + await fs.writeFile(path.join(root, "chunk-a.js"), "alpha"); + await fs.writeFile(path.join(root, "chunk-b.js"), "beta"); + const slowDown = () => Object.assign(new Error("Please reduce your request rate."), { name: "SlowDown", Code: "SlowDown", $metadata: { httpStatusCode: 503 } }); + const put = provider.putObjectConditional!.bind(provider); + let refusals = 3; + provider.putObjectConditional = async input => { + if (refusals > 0) { refusals--; for await (const _chunk of input.body) { /* drain as a real client would */ } throw slowDown(); } + return put(input); + }; + const waits: number[] = []; + await retainedUiAssets(provider, { sleep: async ms => { waits.push(ms); } }).publish(root); + expect(waits).toEqual([500, 1_000, 2_000]); + expect(objects.get("ui-assets/v1/chunk-a.js")?.toString()).toBe("alpha"); + expect(objects.get("ui-assets/v1/chunk-b.js")?.toString()).toBe("beta"); +}); + +it("gives up on storage that stays throttled, and reports it as a storage outage rather than a bad build", async () => { + const root = await directory(), { provider } = storage(); + await fs.writeFile(path.join(root, "chunk-a.js"), "alpha"); + provider.putObjectConditional = async () => { throw Object.assign(new Error("Please reduce your request rate."), { name: "SlowDown", $metadata: { httpStatusCode: 503 } }); }; + const waits: number[] = []; + const failure = await retainedUiAssets(provider, { sleep: async ms => { waits.push(ms); } }).publish(root).catch(error => error); + expect(waits).toHaveLength(7); + expect(isTransientStorageError(failure)).toBe(true); + expect(isTransientStorageError(new Error("Static build asset name collides with different published bytes"))).toBe(false); +}); + +it("starts on local files when storage stays unavailable, and still refuses a build that collides", async () => { + const root = await directory(), { provider } = storage(); + await fs.writeFile(path.join(root, "chunk-a.js"), "alpha"); + const put = provider.putObjectConditional!.bind(provider); + provider.putObjectConditional = async () => { throw Object.assign(new Error("socket hang up"), { cause: Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" }) }); }; + const retained = retainedUiAssets(provider, { sleep: async () => {} }); + const reported: unknown[] = []; + await expect(publishRetainedUiAssetsAtStartup(retained, root, error => { reported.push(error); })).resolves.toBe(false); + expect(reported).toHaveLength(1); + // The server that started anyway serves the build from its own disk, and asks storage for nothing it lacks. + const app = express(); + app.use("/assets", staticUiFiles(root), retained.serve); + await request(app).get("/assets/chunk-a.js").expect(200).expect("alpha"); + await request(app).get("/assets/never-built.js").expect(404); + // Once storage answers, a published name with different bytes is a bad build, not an outage. + provider.putObjectConditional = put; + await expect(publishRetainedUiAssetsAtStartup(retained, root, () => {})).resolves.toBe(true); + await fs.writeFile(path.join(root, "chunk-a.js"), "collision"); + await expect(publishRetainedUiAssetsAtStartup(retained, root, error => { reported.push(error); })).rejects.toThrow("collides"); + expect(reported).toHaveLength(1); +}); + +it("retries an existing asset whose saved bytes storage could not serve, without accepting it unverified", async () => { + const root = await directory(), { provider } = storage(); + await fs.writeFile(path.join(root, "chunk-a.js"), "alpha"); + const waits: number[] = []; + const retained = retainedUiAssets(provider, { sleep: async ms => { waits.push(ms); } }); + await retained.publish(root); + // A restart: the conditional write answers 412 and the verifying read is throttled twice. + const get = provider.getObject.bind(provider); + let refusals = 2, reads = 0; + provider.getObject = async input => { + reads++; + if (refusals > 0) { refusals--; throw Object.assign(new Error("Please reduce your request rate."), { name: "SlowDown", $metadata: { httpStatusCode: 503 } }); } + return get(input); + }; + await retained.publish(root); + expect(waits).toEqual([500, 1_000]); + expect(reads).toBe(3); + // A deadline on the asset is an outage too; a read that says the object is gone is not. + expect(isTransientStorageError(Object.assign(new Error("The operation was aborted"), { name: "AbortError" }))).toBe(true); + expect(isTransientStorageError(Object.assign(new Error("Object missing"), { status: 404 }))).toBe(false); +}); diff --git a/server/src/app.ts b/server/src/app.ts index 1d3192e90ec..11e12eaefd8 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -21,7 +21,7 @@ import { startAgentLifecycle } from "./services/agent-lifecycle.js"; import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js"; import { databaseOperationMiddleware } from "./middleware/database-operation.js"; import { staticUiFiles } from "./middleware/static-ui.js"; -import { retainedUiAssets } from "./retained-ui-assets.js"; +import { publishRetainedUiAssetsAtStartup, retainedUiAssets } from "./retained-ui-assets.js"; import { createStorageProviderFromConfig } from "./storage/provider-registry.js"; import { loadStorageConfig } from "./storage/config.js"; import { isIdleTaskDrainActive, trackIdleWork } from "./services/task-admission.js"; @@ -1107,7 +1107,8 @@ export async function createApp( ); if (process.env.PAPERCLIP_SHARED_UI_ASSETS === "true") { const retained = retainedUiAssets(createStorageProviderFromConfig(loadStorageConfig())); - await retained.publish(path.join(uiDist, "assets")); + await publishRetainedUiAssetsAtStartup(retained, path.join(uiDist, "assets"), error => + logger.error({ err: error }, "Shared UI assets could not be published because storage is unavailable; serving this build from local files")); staticUi.use("/assets", retained.serve); } // Serve root/index through the same runtime HTML transform as SPA routes. diff --git a/server/src/retained-ui-assets.ts b/server/src/retained-ui-assets.ts index 89c618a62ad..5536c4d46da 100644 --- a/server/src/retained-ui-assets.ts +++ b/server/src/retained-ui-assets.ts @@ -19,11 +19,52 @@ async function digest(stream: Readable) { return hash.digest("hex"); } +const transientStatuses = new Set([429, 500, 502, 503, 504]); +const transientNames = new Set(["SlowDown", "Throttling", "ThrottlingException", "RequestTimeout", "ServiceUnavailable", "InternalError", + // The 30-second deadline on one asset, and a connection that never completed. + "TimeoutError", "AbortError"]); +const transientNetworkCodes = new Set(["ECONNREFUSED", "ECONNRESET", "ECONNABORTED", "ETIMEDOUT", "EPIPE", "ENOTFOUND", "EAI_AGAIN", "ENETUNREACH", "EHOSTUNREACH", + "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]); +const publishRetryDelaysMs = [500, 1_000, 2_000, 4_000, 8_000, 8_000, 8_000]; + +/** Storage asked us to slow down or failed on its side; the same request can succeed later. */ +export function isTransientStorageError(error: unknown): boolean { + // The storage client wraps a socket failure; the reason is on the cause. + for (let current: unknown = error, depth = 0; current && typeof current === "object" && depth < 4; depth++) { + const value = current as { name?: unknown; Code?: unknown; code?: unknown; status?: unknown; cause?: unknown; $metadata?: { httpStatusCode?: unknown } }; + if ([value.name, value.Code].some(name => typeof name === "string" && transientNames.has(name)) + || (typeof value.code === "string" && transientNetworkCodes.has(value.code)) + || [value.status, value.$metadata?.httpStatusCode].some(status => typeof status === "number" && transientStatuses.has(status))) return true; + current = value.cause; + } + return false; +} + +/** Publish at startup. Returns false, after saying so, when storage itself is + * unavailable: this container still serves its build from disk, so a storage + * outage costs an older tab its retained chunks, not everyone the server. A + * build that cannot be published for any other reason still rejects. */ +export async function publishRetainedUiAssetsAtStartup( + retained: { publish(directory: string): Promise }, + directory: string, + report: (error: unknown) => void, +): Promise { + try { + await retained.publish(directory); + return true; + } catch (error) { + if (!isTransientStorageError(error)) throw error; + report(error); + return false; + } +} + /** Public, immutable build assets only. Publishing before readiness lets an * older tab fetch its chunks after the last old container has disappeared. * Objects have no automatic expiry: retain them at least 30 days after the * last serving deployment, including rollback, before operator reclamation. */ -export function retainedUiAssets(provider: StorageProvider): { publish(directory: string): Promise; serve: RequestHandler } { +export function retainedUiAssets(provider: StorageProvider, options: { sleep?: (ms: number) => Promise } = {}): { publish(directory: string): Promise; serve: RequestHandler } { + const sleep = options.sleep ?? ((ms: number) => new Promise(resolve => setTimeout(resolve, ms))); if (provider.id !== "s3" || !provider.putObjectConditional) throw new Error("Retained UI assets require conditional shared object storage"); async function publish(directory: string) { async function visit(relativeDirectory: string) { @@ -33,19 +74,33 @@ export function retainedUiAssets(provider: StorageProvider): { publish(directory if (relative.endsWith(".map")) continue; if (!entry.isFile() || !assetPath(relative)) throw new Error("Unsupported static build asset"); const filename = path.join(directory, relative); - const signal = AbortSignal.timeout(30_000); - const contentLength = (await fs.stat(filename)).size; - const expected = await digest(createReadStream(filename, { signal })); const objectKey = `${prefix}${relative}`; - const body = createReadStream(filename, { signal }); - try { - await provider.putObjectConditional!({ objectKey, body, contentLength, contentType: "application/octet-stream", ifNoneMatch: "*", signal }); - } catch (error) { - // An existing key or uncertain upload may be accepted only if its - // complete bytes match. Never overwrite a colliding build asset. - const existing = await provider.getObject({ objectKey, signal }).catch(() => { throw error; }); - if (await digest(existing.stream) !== expected) throw new Error("Static build asset name collides with different published bytes"); - } finally { body.destroy(); } + // A build has hundreds of assets and storage throttles a burst. The + // upload body is a stream, so the client cannot replay it: retry the + // whole asset here, with a fresh stream, instead of failing startup. + for (let attempt = 0; ; attempt++) { + const signal = AbortSignal.timeout(30_000); + const contentLength = (await fs.stat(filename)).size; + const expected = await digest(createReadStream(filename, { signal })); + const body = createReadStream(filename, { signal }); + try { + try { + await provider.putObjectConditional!({ objectKey, body, contentLength, contentType: "application/octet-stream", ifNoneMatch: "*", signal }); + } catch (error) { + // An existing key or uncertain upload may be accepted only if its + // complete bytes match. Never overwrite a colliding build asset. + // A read that storage could not serve proves nothing either way: + // keep it, so the asset is tried again instead of being judged. + const existing = await provider.getObject({ objectKey, signal }) + .catch(readError => { throw isTransientStorageError(readError) ? readError : error; }); + if (await digest(existing.stream) !== expected) throw new Error("Static build asset name collides with different published bytes"); + } + break; + } catch (error) { + if (!isTransientStorageError(error) || attempt >= publishRetryDelaysMs.length) throw error; + await sleep(publishRetryDelaysMs[attempt]!); + } finally { body.destroy(); } + } } } await visit("");