fix(run-log): give each object-journal read attempt its own deadline (#21508)

Give each object-journal read attempt its own deadline.

Co-Authored-By: Paperclip <Paperclip-Paperclip@users.noreply.github.com>
This commit is contained in:
Devin FoleyandPaperclip authored and GitHub committed 2026-11-20 10:31:58 -08:00
1 parent 1a50bd550e
commit 42c8fee59b
2 files changed
+88 -21

No files matched your search

@@ -133,6 +133,65 @@ describe("object byte journal", () => {
expect(isStorageUnavailableError(failure)).toBe(true);
}, 40_000);
it("sends a read again with a fresh deadline when one GET hangs until its deadline", async () => {
const s = storage();
const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung/log" });
const claimed = await journal.claim({ assertAuthority: authority });
const get = s.provider.getObject;
let headReads = 0;
s.provider.getObject = async (input) => {
if (!input.objectKey.endsWith("/head.json") || ++headReads > 1) return get(input);
return withS3FailureDiagnostics("get_object", () => new Promise<never>((_resolve, reject) => {
input.signal!.addEventListener("abort", () => {
const aborted = new Error("Request aborted");
aborted.name = "AbortError";
reject(aborted);
}, { once: true });
}), input.signal);
};
const current = await journal.head();
expect(headReads).toBe(2);
expect(current?.etag).toBe(claimed.etag);
}, 40_000);
it("does not send a hung read again after its caller aborts", async () => {
const s = storage();
const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung-stop/log" });
await journal.claim({ assertAuthority: authority });
let headReads = 0;
s.provider.getObject = (input) => {
headReads++;
return new Promise<never>((_resolve, reject) => {
input.signal!.addEventListener("abort", () => reject(input.signal!.reason), { once: true });
});
};
const controller = new AbortController();
setTimeout(() => controller.abort(new Error("caller stopped")), 100);
await expect(journal.head(controller.signal)).rejects.toThrow("caller stopped");
await new Promise((resolve) => setTimeout(resolve, 500));
expect(headReads).toBe(1);
});
it("stops sending a read that hangs on every attempt and reports it as a storage outage", async () => {
const s = storage();
const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung-always/log" });
await journal.claim({ assertAuthority: authority });
let headReads = 0;
s.provider.getObject = (input) => {
headReads++;
return withS3FailureDiagnostics("get_object", () => new Promise<never>((_resolve, reject) => {
input.signal!.addEventListener("abort", () => {
const aborted = new Error("Request aborted");
aborted.name = "AbortError";
reject(aborted);
}, { once: true });
}), input.signal);
};
const failure = await journal.head().catch((error: unknown) => error);
expect(headReads).toBe(3);
expect(isStorageUnavailableError(failure)).toBe(true);
}, 70_000);
it("keeps a shared capability probe alive for another writer when one claim cancels", async () => {
const s = storage();
const put = s.provider.putObjectConditional!;
+29 -21
View File
@@ -69,10 +69,14 @@ export function isObjectPreconditionFailure(error: unknown) {
const e = error as { name?: string; $metadata?: { httpStatusCode?: number } };
return e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412;
}
// Reads abandoned by their own attempt deadline, not by their caller. A GET is
// safe to send again.
const expiredReads = new WeakSet<object>();
function isRetryable(error: unknown) {
const e = error as { name?: string; code?: string; $metadata?: { httpStatusCode?: number } };
const status = e?.$metadata?.httpStatusCode;
return (
(typeof error === "object" && error !== null && expiredReads.has(error)) ||
e?.name === "ConditionalRequestConflict" ||
status === 409 ||
status === 429 ||
@@ -464,30 +468,34 @@ export function createObjectByteJournal(options: ObjectByteJournalOptions) {
async function getBytes(key: string, limit: number, signal?: AbortSignal) {
return objectIO(async () => {
signal?.throwIfAborted();
signal = signal
? AbortSignal.any([signal, AbortSignal.timeout(15_000)])
: AbortSignal.timeout(15_000);
const read = signal;
return sendThroughStorageOutage(async () => {
const result = await retryObjectOperation(
() => provider.getObject({ objectKey: key, signal: read }),
read,
);
const chunks: Buffer[] = [];
let size = 0;
const caller = signal;
return sendThroughStorageOutage(() => retryObjectOperation(async () => {
// Each attempt has its own deadline. One deadline shared by every
// attempt made a single hung GET final: once it expired, nothing
// could send the read again.
const deadline = AbortSignal.timeout(15_000);
const read = caller ? AbortSignal.any([caller, deadline]) : deadline;
try {
for await (const chunk of result.stream) {
const bytes = Buffer.from(chunk);
size += bytes.length;
if (size > limit) throw new Error("object_journal_object_too_large");
chunks.push(bytes);
const result = await provider.getObject({ objectKey: key, signal: read });
const chunks: Buffer[] = [];
let size = 0;
try {
for await (const chunk of result.stream) {
const bytes = Buffer.from(chunk);
size += bytes.length;
if (size > limit) throw new Error("object_journal_object_too_large");
chunks.push(bytes);
}
} finally {
result.stream.destroy();
}
} finally {
result.stream.destroy();
read.throwIfAborted();
return { bytes: Buffer.concat(chunks), etag: result.etag };
} catch (error) {
if (deadline.aborted && !caller?.aborted && error && typeof error === "object") expiredReads.add(error);
throw error;
}
read?.throwIfAborted();
return { bytes: Buffer.concat(chunks), etag: result.etag };
}, read);
}, caller), caller);
}, readPriority, signal);
}
async function head(signal?: AbortSignal): Promise<ObjectJournalWriter | null> {