fix(server): shed only retryable object-journal reads under load (#19176)

Client polling can exhaust the shared object journal queue and reject log persistence. Bound ordinary object-log and trace reads, keep persistence and internal verification queued, and return retryable 503 responses when reads are busy. Leave feedback exports unsent until every log page is available.

Validation: 72 latest-master affected tests and 10 workflow tests pass locally, with the workflow fixture passing strict TypeScript checks. The preceding broader run passed 280 server tests with two opt-in S3 skips. All applicable checks on d0a937768d are green (56 success, three skipped); exact-head Apex review is 5/5 with no unresolved findings. Background persistence still has no fixed queue bound during a prolonged storage outage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
authored and GitHub committed 2026-11-06 13:47:25 -08:00
1 parent 8d525dba7d
commit e482f0f632
18 files changed
+382 -33

No files matched your search

@@ -309,7 +309,7 @@ export async function nativeRunLogPrefixCursor(input: {
while (offset < bytes) {
input.signal?.throwIfAborted();
const page = await input.store.read(input.handle, {
offset, limitBytes: Math.min(256_000, bytes - offset), signal: input.signal,
offset, limitBytes: Math.min(256_000, bytes - offset), signal: input.signal, priority: "background",
});
const content = Buffer.from(page.content);
if (!content.length || offset + content.length > bytes) throw unproven();
@@ -0,0 +1,20 @@
import { describe, expect, it, vi } from "vitest";
import type { Db } from "@paperclipai/db";
import type { RunLogStore } from "../run-log-store.js";
import { readRecoverableRunLog } from "./native-run-log-recovery.js";
describe("recoverable run log reads", () => {
it("lets the client log poll be shed under load and keeps the caller's options", async () => {
const read = vi.fn(async () => ({ content: "", nextOffset: 0 }));
const store = { read } as unknown as RunLogStore;
const handle = { store: "local_file" as const, logRef: "logs/run.ndjson" };
const signal = new AbortController().signal;
await readRecoverableRunLog({} as Db, store, handle, { offset: 12, limitBytes: 64, signal });
expect(read).toHaveBeenCalledWith(handle, {
priority: "interactive",
offset: 12,
limitBytes: 64,
signal,
});
});
});
@@ -73,7 +73,7 @@ export async function readRecoverableRunLog(
);
},
);
return store.read(handle, options);
return store.read(handle, { priority: "interactive", ...options });
}
export async function recoverSettledNativeRunLog(