diff --git a/server/src/services/object-byte-journal.test.ts b/server/src/services/object-byte-journal.test.ts index aab25cef77b..7063ca0c748 100644 --- a/server/src/services/object-byte-journal.test.ts +++ b/server/src/services/object-byte-journal.test.ts @@ -133,6 +133,65 @@ describe("object byte journal", () => { expect(isStorageUnavailableError(failure)).toBe(true); }, 40_000); + it("sends a read again with a fresh deadline when one GET hangs until its deadline", async () => { + const s = storage(); + const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung/log" }); + const claimed = await journal.claim({ assertAuthority: authority }); + const get = s.provider.getObject; + let headReads = 0; + s.provider.getObject = async (input) => { + if (!input.objectKey.endsWith("/head.json") || ++headReads > 1) return get(input); + return withS3FailureDiagnostics("get_object", () => new Promise((_resolve, reject) => { + input.signal!.addEventListener("abort", () => { + const aborted = new Error("Request aborted"); + aborted.name = "AbortError"; + reject(aborted); + }, { once: true }); + }), input.signal); + }; + const current = await journal.head(); + expect(headReads).toBe(2); + expect(current?.etag).toBe(claimed.etag); + }, 40_000); + + it("does not send a hung read again after its caller aborts", async () => { + const s = storage(); + const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung-stop/log" }); + await journal.claim({ assertAuthority: authority }); + let headReads = 0; + s.provider.getObject = (input) => { + headReads++; + return new Promise((_resolve, reject) => { + input.signal!.addEventListener("abort", () => reject(input.signal!.reason), { once: true }); + }); + }; + const controller = new AbortController(); + setTimeout(() => controller.abort(new Error("caller stopped")), 100); + await expect(journal.head(controller.signal)).rejects.toThrow("caller stopped"); + await new Promise((resolve) => setTimeout(resolve, 500)); + expect(headReads).toBe(1); + }); + + it("stops sending a read that hangs on every attempt and reports it as a storage outage", async () => { + const s = storage(); + const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung-always/log" }); + await journal.claim({ assertAuthority: authority }); + let headReads = 0; + s.provider.getObject = (input) => { + headReads++; + return withS3FailureDiagnostics("get_object", () => new Promise((_resolve, reject) => { + input.signal!.addEventListener("abort", () => { + const aborted = new Error("Request aborted"); + aborted.name = "AbortError"; + reject(aborted); + }, { once: true }); + }), input.signal); + }; + const failure = await journal.head().catch((error: unknown) => error); + expect(headReads).toBe(3); + expect(isStorageUnavailableError(failure)).toBe(true); + }, 70_000); + it("keeps a shared capability probe alive for another writer when one claim cancels", async () => { const s = storage(); const put = s.provider.putObjectConditional!; diff --git a/server/src/services/object-byte-journal.ts b/server/src/services/object-byte-journal.ts index 88d12fb1067..fc864a62ca1 100644 --- a/server/src/services/object-byte-journal.ts +++ b/server/src/services/object-byte-journal.ts @@ -69,10 +69,14 @@ export function isObjectPreconditionFailure(error: unknown) { const e = error as { name?: string; $metadata?: { httpStatusCode?: number } }; return e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412; } +// Reads abandoned by their own attempt deadline, not by their caller. A GET is +// safe to send again. +const expiredReads = new WeakSet(); function isRetryable(error: unknown) { const e = error as { name?: string; code?: string; $metadata?: { httpStatusCode?: number } }; const status = e?.$metadata?.httpStatusCode; return ( + (typeof error === "object" && error !== null && expiredReads.has(error)) || e?.name === "ConditionalRequestConflict" || status === 409 || status === 429 || @@ -464,30 +468,34 @@ export function createObjectByteJournal(options: ObjectByteJournalOptions) { async function getBytes(key: string, limit: number, signal?: AbortSignal) { return objectIO(async () => { signal?.throwIfAborted(); - signal = signal - ? AbortSignal.any([signal, AbortSignal.timeout(15_000)]) - : AbortSignal.timeout(15_000); - const read = signal; - return sendThroughStorageOutage(async () => { - const result = await retryObjectOperation( - () => provider.getObject({ objectKey: key, signal: read }), - read, - ); - const chunks: Buffer[] = []; - let size = 0; + const caller = signal; + return sendThroughStorageOutage(() => retryObjectOperation(async () => { + // Each attempt has its own deadline. One deadline shared by every + // attempt made a single hung GET final: once it expired, nothing + // could send the read again. + const deadline = AbortSignal.timeout(15_000); + const read = caller ? AbortSignal.any([caller, deadline]) : deadline; try { - for await (const chunk of result.stream) { - const bytes = Buffer.from(chunk); - size += bytes.length; - if (size > limit) throw new Error("object_journal_object_too_large"); - chunks.push(bytes); + const result = await provider.getObject({ objectKey: key, signal: read }); + const chunks: Buffer[] = []; + let size = 0; + try { + for await (const chunk of result.stream) { + const bytes = Buffer.from(chunk); + size += bytes.length; + if (size > limit) throw new Error("object_journal_object_too_large"); + chunks.push(bytes); + } + } finally { + result.stream.destroy(); } - } finally { - result.stream.destroy(); + read.throwIfAborted(); + return { bytes: Buffer.concat(chunks), etag: result.etag }; + } catch (error) { + if (deadline.aborted && !caller?.aborted && error && typeof error === "object") expiredReads.add(error); + throw error; } - read?.throwIfAborted(); - return { bytes: Buffer.concat(chunks), etag: result.etag }; - }, read); + }, caller), caller); }, readPriority, signal); } async function head(signal?: AbortSignal): Promise {