fix(server): shed only retryable object-journal reads under load (#19176)

Client polling can exhaust the shared object journal queue and reject log persistence. Bound ordinary object-log and trace reads, keep persistence and internal verification queued, and return retryable 503 responses when reads are busy. Leave feedback exports unsent until every log page is available.

Validation: 72 latest-master affected tests and 10 workflow tests pass locally, with the workflow fixture passing strict TypeScript checks. The preceding broader run passed 280 server tests with two opt-in S3 skips. All applicable checks on d0a937768d are green (56 success, three skipped); exact-head Apex review is 5/5 with no unresolved findings. Background persistence still has no fixed queue bound during a prolonged storage outage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
authored and GitHub committed 2026-11-06 13:47:25 -08:00
1 parent 8d525dba7d
commit e482f0f632
18 files changed
+382 -33

No files matched your search

@@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { HttpError } from "../errors.js";
import { errorHandler } from "../middleware/error-handler.js";
import { mcpDiscoveryHttpFailure, retainMcpConnectionFailure } from "../services/mcp-connection-failure.js";
import { ObjectJournalBusyError } from "../services/object-byte-journal.js";
const recordResponsibleUserDenialOnActiveRunMock = vi.hoisted(() => vi.fn());
const captureExceptionMock = vi.hoisted(() => vi.fn());
@@ -115,6 +116,23 @@ describe("errorHandler", () => {
expect(telemetryMocks.trackErrorHandlerCrash).not.toHaveBeenCalled();
});
it("answers a shed object-storage read with a retryable 503 without reporting it", () => {
const res = { ...makeRes(), set: vi.fn() } as any;
const next = vi.fn() as unknown as NextFunction;
errorHandler(new ObjectJournalBusyError(), makeReq(), res, next);
expect(res.set).toHaveBeenCalledWith("Retry-After", "2");
expect(res.status).toHaveBeenCalledWith(503);
expect(res.json).toHaveBeenCalledWith({
error: "Object storage is busy; retry shortly",
code: "object_journal_io_busy",
});
expect(res.err).toBeUndefined();
expect(captureExceptionMock).not.toHaveBeenCalled();
expect(telemetryMocks.trackErrorHandlerCrash).not.toHaveBeenCalled();
});
it("only ends the response when headers were already sent during database shutdown", () => {
const req = { ...makeReq(), app: { locals: { paperclipDatabaseClosing: true } } } as unknown as Request;
const res = { ...makeRes(), end: vi.fn(), headersSent: true } as any;