mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-12 22:56:58 +01:00
fix(server): shed only retryable object-journal reads under load (#19176)
Client polling can exhaust the shared object journal queue and reject log persistence. Bound ordinary object-log and trace reads, keep persistence and internal verification queued, and return retryable 503 responses when reads are busy. Leave feedback exports unsent until every log page is available.
Validation: 72 latest-master affected tests and 10 workflow tests pass locally, with the workflow fixture passing strict TypeScript checks. The preceding broader run passed 280 server tests with two opt-in S3 skips. All applicable checks on d0a937768d are green (56 success, three skipped); exact-head Apex review is 5/5 with no unresolved findings. Background persistence still has no fixed queue bound during a prolonged storage outage.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
18 files changed
+382
-33
No files matched your search
@@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { HttpError } from "../errors.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { mcpDiscoveryHttpFailure, retainMcpConnectionFailure } from "../services/mcp-connection-failure.js";
|
||||
import { ObjectJournalBusyError } from "../services/object-byte-journal.js";
|
||||
|
||||
const recordResponsibleUserDenialOnActiveRunMock = vi.hoisted(() => vi.fn());
|
||||
const captureExceptionMock = vi.hoisted(() => vi.fn());
|
||||
@@ -115,6 +116,23 @@ describe("errorHandler", () => {
|
||||
expect(telemetryMocks.trackErrorHandlerCrash).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("answers a shed object-storage read with a retryable 503 without reporting it", () => {
|
||||
const res = { ...makeRes(), set: vi.fn() } as any;
|
||||
const next = vi.fn() as unknown as NextFunction;
|
||||
|
||||
errorHandler(new ObjectJournalBusyError(), makeReq(), res, next);
|
||||
|
||||
expect(res.set).toHaveBeenCalledWith("Retry-After", "2");
|
||||
expect(res.status).toHaveBeenCalledWith(503);
|
||||
expect(res.json).toHaveBeenCalledWith({
|
||||
error: "Object storage is busy; retry shortly",
|
||||
code: "object_journal_io_busy",
|
||||
});
|
||||
expect(res.err).toBeUndefined();
|
||||
expect(captureExceptionMock).not.toHaveBeenCalled();
|
||||
expect(telemetryMocks.trackErrorHandlerCrash).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("only ends the response when headers were already sent during database shutdown", () => {
|
||||
const req = { ...makeReq(), app: { locals: { paperclipDatabaseClosing: true } } } as unknown as Request;
|
||||
const res = { ...makeRes(), end: vi.fn(), headersSent: true } as any;
|
||||
|
||||
Reference in new issue
Block a user