mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-21 02:43:25 +01:00
fix(run-log): give each object-journal read attempt its own deadline (#21508)
Give each object-journal read attempt its own deadline. Co-Authored-By: Paperclip <Paperclip-Paperclip@users.noreply.github.com>
This commit is contained in:
1 parent
1a50bd550e
commit
42c8fee59b
2 files changed
+88
-21
No files matched your search
@@ -133,6 +133,65 @@ describe("object byte journal", () => {
|
||||
expect(isStorageUnavailableError(failure)).toBe(true);
|
||||
}, 40_000);
|
||||
|
||||
it("sends a read again with a fresh deadline when one GET hangs until its deadline", async () => {
|
||||
const s = storage();
|
||||
const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung/log" });
|
||||
const claimed = await journal.claim({ assertAuthority: authority });
|
||||
const get = s.provider.getObject;
|
||||
let headReads = 0;
|
||||
s.provider.getObject = async (input) => {
|
||||
if (!input.objectKey.endsWith("/head.json") || ++headReads > 1) return get(input);
|
||||
return withS3FailureDiagnostics("get_object", () => new Promise<never>((_resolve, reject) => {
|
||||
input.signal!.addEventListener("abort", () => {
|
||||
const aborted = new Error("Request aborted");
|
||||
aborted.name = "AbortError";
|
||||
reject(aborted);
|
||||
}, { once: true });
|
||||
}), input.signal);
|
||||
};
|
||||
const current = await journal.head();
|
||||
expect(headReads).toBe(2);
|
||||
expect(current?.etag).toBe(claimed.etag);
|
||||
}, 40_000);
|
||||
|
||||
it("does not send a hung read again after its caller aborts", async () => {
|
||||
const s = storage();
|
||||
const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung-stop/log" });
|
||||
await journal.claim({ assertAuthority: authority });
|
||||
let headReads = 0;
|
||||
s.provider.getObject = (input) => {
|
||||
headReads++;
|
||||
return new Promise<never>((_resolve, reject) => {
|
||||
input.signal!.addEventListener("abort", () => reject(input.signal!.reason), { once: true });
|
||||
});
|
||||
};
|
||||
const controller = new AbortController();
|
||||
setTimeout(() => controller.abort(new Error("caller stopped")), 100);
|
||||
await expect(journal.head(controller.signal)).rejects.toThrow("caller stopped");
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
expect(headReads).toBe(1);
|
||||
});
|
||||
|
||||
it("stops sending a read that hangs on every attempt and reports it as a storage outage", async () => {
|
||||
const s = storage();
|
||||
const journal = createObjectByteJournal({ provider: s.provider, prefix: "company/hung-always/log" });
|
||||
await journal.claim({ assertAuthority: authority });
|
||||
let headReads = 0;
|
||||
s.provider.getObject = (input) => {
|
||||
headReads++;
|
||||
return withS3FailureDiagnostics("get_object", () => new Promise<never>((_resolve, reject) => {
|
||||
input.signal!.addEventListener("abort", () => {
|
||||
const aborted = new Error("Request aborted");
|
||||
aborted.name = "AbortError";
|
||||
reject(aborted);
|
||||
}, { once: true });
|
||||
}), input.signal);
|
||||
};
|
||||
const failure = await journal.head().catch((error: unknown) => error);
|
||||
expect(headReads).toBe(3);
|
||||
expect(isStorageUnavailableError(failure)).toBe(true);
|
||||
}, 70_000);
|
||||
|
||||
it("keeps a shared capability probe alive for another writer when one claim cancels", async () => {
|
||||
const s = storage();
|
||||
const put = s.provider.putObjectConditional!;
|
||||
|
||||
@@ -69,10 +69,14 @@ export function isObjectPreconditionFailure(error: unknown) {
|
||||
const e = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
||||
return e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412;
|
||||
}
|
||||
// Reads abandoned by their own attempt deadline, not by their caller. A GET is
|
||||
// safe to send again.
|
||||
const expiredReads = new WeakSet<object>();
|
||||
function isRetryable(error: unknown) {
|
||||
const e = error as { name?: string; code?: string; $metadata?: { httpStatusCode?: number } };
|
||||
const status = e?.$metadata?.httpStatusCode;
|
||||
return (
|
||||
(typeof error === "object" && error !== null && expiredReads.has(error)) ||
|
||||
e?.name === "ConditionalRequestConflict" ||
|
||||
status === 409 ||
|
||||
status === 429 ||
|
||||
@@ -464,30 +468,34 @@ export function createObjectByteJournal(options: ObjectByteJournalOptions) {
|
||||
async function getBytes(key: string, limit: number, signal?: AbortSignal) {
|
||||
return objectIO(async () => {
|
||||
signal?.throwIfAborted();
|
||||
signal = signal
|
||||
? AbortSignal.any([signal, AbortSignal.timeout(15_000)])
|
||||
: AbortSignal.timeout(15_000);
|
||||
const read = signal;
|
||||
return sendThroughStorageOutage(async () => {
|
||||
const result = await retryObjectOperation(
|
||||
() => provider.getObject({ objectKey: key, signal: read }),
|
||||
read,
|
||||
);
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
const caller = signal;
|
||||
return sendThroughStorageOutage(() => retryObjectOperation(async () => {
|
||||
// Each attempt has its own deadline. One deadline shared by every
|
||||
// attempt made a single hung GET final: once it expired, nothing
|
||||
// could send the read again.
|
||||
const deadline = AbortSignal.timeout(15_000);
|
||||
const read = caller ? AbortSignal.any([caller, deadline]) : deadline;
|
||||
try {
|
||||
for await (const chunk of result.stream) {
|
||||
const bytes = Buffer.from(chunk);
|
||||
size += bytes.length;
|
||||
if (size > limit) throw new Error("object_journal_object_too_large");
|
||||
chunks.push(bytes);
|
||||
const result = await provider.getObject({ objectKey: key, signal: read });
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for await (const chunk of result.stream) {
|
||||
const bytes = Buffer.from(chunk);
|
||||
size += bytes.length;
|
||||
if (size > limit) throw new Error("object_journal_object_too_large");
|
||||
chunks.push(bytes);
|
||||
}
|
||||
} finally {
|
||||
result.stream.destroy();
|
||||
}
|
||||
} finally {
|
||||
result.stream.destroy();
|
||||
read.throwIfAborted();
|
||||
return { bytes: Buffer.concat(chunks), etag: result.etag };
|
||||
} catch (error) {
|
||||
if (deadline.aborted && !caller?.aborted && error && typeof error === "object") expiredReads.add(error);
|
||||
throw error;
|
||||
}
|
||||
read?.throwIfAborted();
|
||||
return { bytes: Buffer.concat(chunks), etag: result.etag };
|
||||
}, read);
|
||||
}, caller), caller);
|
||||
}, readPriority, signal);
|
||||
}
|
||||
async function head(signal?: AbortSignal): Promise<ObjectJournalWriter | null> {
|
||||
|
||||
Reference in new issue
Block a user