Files
PaperClipAI/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts
T
Priya RamanandPaperclip fa14631ba2 fix(adapter-utils): charge the duplex gateway body reserve for its true unknown-length peak
The duplex gateway charged an unknown-length request body one time,
but readBody retains a chunk list and the Buffer.concat result
together for one instant, so its true retained peak is up to two
times the body cap. Name that factor as
DUPLEX_GATEWAY_UNKNOWN_LENGTH_BODY_RESERVE_FACTOR and charge
2 * maxBodyBytes plus the fixed frame allowance for an unknown-length
read. A declared-length read still charges the declared length plus
the same fixed allowance, and a bodyless request still charges the
zero-length reserve. Update the retained-peak comment to state the
true unknown-length peak.

The file gateway reserve is unchanged: FILE_GATEWAY_BODY_RESERVE_FACTOR
stays 4.

Add a regression test that sends four concurrent cap-sized chunked
requests with no Content-Length header, holds the host forward so
the reserves overlap, and sets the bound to admit exactly three
reserves of 2 * maxBodyBytes + the frame allowance. It asserts the
fourth request gets a 503 bridge_busy, then asserts a later request
is admitted once a terminal releases a reserve. Proven against the
pre-fix commit first: the test fails there (all four requests admit,
none shed) and passes after this fix.

Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-08-23 21:47:28 +00:00

1466 lines
59 KiB
TypeScript

import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import net from "node:net";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { Readable } from "node:stream";
import { StringDecoder } from "node:string_decoder";
import { afterEach, describe, expect, it } from "vitest";
import {
authorizeSandboxCallbackBridgeRequestWithRoutes,
getSandboxCallbackBridgeServerSource,
SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE,
} from "./sandbox-callback-bridge.js";
import {
DuplexFrameDecoder,
DEFAULT_MAX_DUPLEX_FRAME_BYTES,
DUPLEX_BODY_CHUNK_RAW_BYTES,
DUPLEX_FRAME_VERSION,
encodeDuplexFrame,
type DuplexBodyChunkFrame,
type DuplexFrame,
type DuplexReadyFrame,
type DuplexRequestFrame,
} from "./duplex-frame-codec.js";
import {
createDuplexBridgeBroker,
type DuplexBridgeBroker,
type DuplexBrokerForwardResult,
} from "./duplex-bridge-broker.js";
import type { ReassembledBody } from "./duplex-body-spool.js";
import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js";
/**
* Local real-process end-to-end harness for the composed duplex path.
*
* The harness spawns the real generated gateway with plain `node` from a
* temporary file, attaches the real host broker to the child stdin and stdout,
* and forwards each request to a local fake API server on a real HTTP call. It
* uses no provider credentials.
*
* The child stdio pipes are kernel pipes. They give partial reads, split a
* multi-byte UTF-8 sequence across two chunks, apply backpressure, and give a
* true end of file when the child dies. The harness proves the composed path
* handles each of these real conditions.
*/
/** One recorded call the fake API server received. */
interface FakeApiRequest {
method: string;
url: string;
auth: string | null;
runId: string | null;
body: string;
}
/** The responder the fake API server calls for each request. */
type FakeApiResponder = (req: IncomingMessage, res: ServerResponse, body: string) => void;
/** The handle for one fake API server. */
interface FakeApiServer {
origin: string;
requests: FakeApiRequest[];
setResponder: (responder: FakeApiResponder) => void;
/** The count of sockets the server holds open right now. */
openSocketCount: () => number;
/** True while the server still accepts connections. */
listening: () => boolean;
close: () => Promise<void>;
}
/**
* Start a local fake API server. The forward handler targets it, so one real
* HTTP call proves the composed path end to end. The server tracks each open
* socket, so a test asserts teardown leaves no open socket handle.
*/
async function startFakeApiServer(): Promise<FakeApiServer> {
const requests: FakeApiRequest[] = [];
const sockets = new Set<net.Socket>();
let responder: FakeApiResponder = (req, res) => {
const url = new URL(req.url ?? "/", "http://127.0.0.1");
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true, path: url.pathname }));
};
const server = createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk: Buffer) => chunks.push(chunk));
req.on("end", () => {
const body = Buffer.concat(chunks).toString("utf8");
requests.push({
method: req.method ?? "GET",
url: req.url ?? "/",
auth: req.headers.authorization ?? null,
runId:
typeof req.headers["x-paperclip-run-id"] === "string"
? req.headers["x-paperclip-run-id"]
: null,
body,
});
responder(req, res, body);
});
});
server.on("connection", (socket) => {
sockets.add(socket);
socket.on("close", () => sockets.delete(socket));
});
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => resolve());
});
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("The fake API server did not expose a TCP port.");
}
return {
origin: `http://127.0.0.1:${address.port}`,
requests,
setResponder: (next) => {
responder = next;
},
openSocketCount: () => sockets.size,
listening: () => server.listening,
close: () =>
new Promise<void>((resolve) => {
// Destroy each open socket first. A keep-alive client socket keeps the
// server open, so `server.close` alone could stall. The destroy makes the
// close callback fire and drives the open socket count to zero.
for (const socket of sockets) socket.destroy();
server.close(() => resolve());
}),
};
}
/** The channel view over the spawned child, plus the frames the child sent host-ward. */
interface ChildDuplexChannel {
channel: CommandManagedDuplexChannel;
observedFrames: DuplexFrame[];
stderr: () => string;
}
/**
* Wrap the spawned child as a {@link CommandManagedDuplexChannel}. The broker
* writes to the child stdin, reads the child stdout, and learns of the child
* exit through this channel.
*
* The host end reads a byte stream from the stdout pipe. A pipe read can split
* one multi-byte UTF-8 character across two chunks. The `StringDecoder` holds
* the bytes of an incomplete character until the next chunk, so the broker only
* ever reads whole characters. The channel keeps a second decoder for
* observation only; it lets the harness assert the READY frame and the request
* frames the child produced, and it never feeds the broker.
*/
function attachChildDuplexChannel(child: ChildProcessWithoutNullStreams): ChildDuplexChannel {
const observed = new DuplexFrameDecoder();
const observedFrames: DuplexFrame[] = [];
const stdoutDecoder = new StringDecoder("utf8");
let dataListener: ((chunk: string) => void) | null = null;
let exitListener: ((exit: { exitCode: number | null }) => void) | null = null;
let pendingText = "";
let pendingExit: { exitCode: number | null } | null = null;
let stderrText = "";
child.stdout.on("data", (buffer: Buffer) => {
for (const result of observed.push(buffer)) {
if (result.ok) observedFrames.push(result.frame);
}
const text = stdoutDecoder.write(buffer);
if (text.length === 0) return;
if (dataListener) dataListener(text);
else pendingText += text;
});
child.stderr.on("data", (buffer: Buffer) => {
stderrText += buffer.toString("utf8");
});
child.on("exit", (code) => {
const exit = { exitCode: code };
if (exitListener) exitListener(exit);
else pendingExit = exit;
});
// Swallow a stdin EPIPE. The broker can write one more frame while the child
// exits; the write fails and the broker records the loss on its own path.
child.stdin.on("error", () => undefined);
const channel: CommandManagedDuplexChannel = {
write: (data) => {
child.stdin.write(data);
},
onData: (listener) => {
dataListener = listener;
if (pendingText.length > 0) {
const replay = pendingText;
pendingText = "";
listener(replay);
}
},
onExit: (listener) => {
exitListener = listener;
if (pendingExit) {
const exit = pendingExit;
pendingExit = null;
listener(exit);
}
},
stop: () => {
child.kill("SIGKILL");
},
close: () =>
new Promise<void>((resolve) => {
// Close the write side. The child stdin reaches end of file, so the
// gateway sees a real EOF.
child.stdin.end(() => resolve());
}),
};
return { channel, observedFrames, stderr: () => stderrText };
}
/** The forward-handler mode. `proxy` calls the fake API; `hang` blocks until abort. */
type ForwardMode = "proxy" | "hang";
/** The options for one harness. */
interface HarnessOptions {
hostApiToken?: string;
runId?: string;
maxBodyBytes?: number;
maxInflightBodyBytes?: number;
responseTimeoutMs?: number;
lossExitGraceMs?: number;
}
/** The full harness handle for one composed duplex path. */
interface DuplexE2EHarness {
baseUrl: string;
bridgeToken: string;
hostApiToken: string;
runId: string;
nonce: string;
broker: DuplexBridgeBroker;
api: FakeApiServer;
child: ChildProcessWithoutNullStreams;
observedFrames: DuplexFrame[];
forwardedRequests: DuplexRequestFrame[];
setForwardMode: (mode: ForwardMode) => void;
stderr: () => string;
killChild: () => Promise<void>;
waitFor: (predicate: () => boolean, message: string, timeoutMs?: number) => Promise<void>;
teardown: () => Promise<void>;
}
/** Reserve one free loopback port. The host assigns it to the gateway. */
async function reserveLoopbackPort(): Promise<number> {
return new Promise<number>((resolve, reject) => {
const probe = net.createServer();
probe.once("error", reject);
probe.listen(0, "127.0.0.1", () => {
const address = probe.address();
if (!address || typeof address === "string") {
probe.close(() => reject(new Error("Could not reserve a loopback port.")));
return;
}
const reserved = address.port;
probe.close(() => resolve(reserved));
});
});
}
/**
* Build the composed duplex path: a spawned gateway child, the real broker on
* the child stdio, and a local fake API server as the forward target.
*/
async function createHarness(options: HarnessOptions = {}): Promise<DuplexE2EHarness> {
const hostApiToken = options.hostApiToken ?? "real-run-jwt";
const runId = options.runId ?? "run-e2e";
const maxBodyBytes = options.maxBodyBytes ?? 1_000_000;
const bridgeToken = "duplex-e2e-bridge-token";
const nonce = "e2e112233445566778899aabbccddeeff";
let forwardMode: ForwardMode = "proxy";
const forwardedRequests: DuplexRequestFrame[] = [];
const api = await startFakeApiServer();
const assignedPort = await reserveLoopbackPort();
const tmpDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-e2e-"));
const entrypoint = path.join(tmpDir, "gateway.mjs");
await writeFile(entrypoint, getSandboxCallbackBridgeServerSource(), "utf8");
const child = spawn(process.execPath, [entrypoint], {
stdio: ["pipe", "pipe", "pipe"],
env: {
...process.env,
PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE,
PAPERCLIP_BRIDGE_HOST: "127.0.0.1",
PAPERCLIP_BRIDGE_PORT: String(assignedPort),
PAPERCLIP_BRIDGE_NONCE: nonce,
PAPERCLIP_BRIDGE_TOKEN: bridgeToken,
PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes),
...(options.maxInflightBodyBytes != null
? { PAPERCLIP_BRIDGE_MAX_INFLIGHT_BODY_BYTES: String(options.maxInflightBodyBytes) }
: {}),
...(options.responseTimeoutMs != null
? { PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS: String(options.responseTimeoutMs) }
: {}),
...(options.lossExitGraceMs != null
? { PAPERCLIP_BRIDGE_LOSS_EXIT_GRACE_MS: String(options.lossExitGraceMs) }
: {}),
},
}) as ChildProcessWithoutNullStreams;
const { channel, observedFrames, stderr } = attachChildDuplexChannel(child);
const forwardRequest = async (
request: DuplexRequestFrame,
opts: { signal: AbortSignal; body: ReassembledBody },
): Promise<DuplexBrokerForwardResult> => {
forwardedRequests.push(request);
// Keep the real route allowlist on the forward seam. The broker forwards
// only an allowed route; it denies any other route with a 403.
const denial = authorizeSandboxCallbackBridgeRequestWithRoutes(request);
if (denial) {
return {
status: 403,
headers: { "content-type": "application/json" },
body: JSON.stringify({ error: denial }),
};
}
if (forwardMode === "hang") {
// Block until the broker aborts the forward. This lets a test hold an
// outstanding request open while it forces a loss.
await new Promise<never>((_resolve, reject) => {
const onAbort = () => reject(new Error("The forward call was aborted."));
if (opts.signal.aborted) {
onAbort();
return;
}
opts.signal.addEventListener("abort", onAbort, { once: true });
});
}
const method = request.method.trim().toUpperCase() || "GET";
const headers = new Headers();
for (const [key, value] of Object.entries(request.headers)) {
if (value.trim().length === 0) continue;
headers.set(key, value);
}
// Apply the real host token and the run id, the same as the file bridge path.
headers.set("authorization", `Bearer ${hostApiToken}`);
headers.set("x-paperclip-run-id", runId);
const target = new URL(`${request.path}${request.query ?? ""}`, api.origin);
// Stream the reassembled request body to the host, the same as the real
// forward. A streamed body needs `duplex: "half"`.
const forwardInit: RequestInit & { duplex?: "half" } = { method, headers, signal: opts.signal };
if (method !== "GET" && method !== "HEAD") {
forwardInit.body = Readable.toWeb(
opts.body.createReadStream(),
) as unknown as ReadableStream<Uint8Array>;
forwardInit.duplex = "half";
}
const response = await fetch(target, forwardInit);
const body = await response.text();
const outHeaders: Record<string, string> = {};
response.headers.forEach((value, key) => {
if (key.toLowerCase() === "content-length") return;
outHeaders[key] = value;
});
return { status: response.status, headers: outHeaders, body };
};
const broker = await createDuplexBridgeBroker({
channel,
forwardRequest,
logger: () => undefined,
});
broker.start();
const waitFor = async (
predicate: () => boolean,
message: string,
timeoutMs = 5000,
): Promise<void> => {
const deadline = Date.now() + timeoutMs;
for (;;) {
if (predicate()) return;
if (Date.now() > deadline) {
throw new Error(`Timed out waiting for ${message}. stderr: ${stderr()}`);
}
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, 20);
timer.unref?.();
});
}
};
const waitForExit = (): Promise<void> =>
new Promise<void>((resolve) => {
if (child.exitCode !== null || child.signalCode !== null) {
resolve();
return;
}
child.once("exit", () => resolve());
});
const killChild = async (): Promise<void> => {
child.kill("SIGKILL");
await waitForExit();
};
let torndown = false;
const teardown = async (): Promise<void> => {
if (torndown) return;
torndown = true;
child.kill("SIGKILL");
await waitForExit();
child.stdin.destroy();
child.stdout.destroy();
child.stderr.destroy();
await api.close();
await rm(tmpDir, { recursive: true, force: true });
};
return {
baseUrl: `http://127.0.0.1:${assignedPort}`,
bridgeToken,
hostApiToken,
runId,
nonce,
broker,
api,
child,
observedFrames,
forwardedRequests,
setForwardMode: (mode) => {
forwardMode = mode;
},
stderr,
killChild,
waitFor,
teardown,
};
}
/**
* Build a large body of multi-byte UTF-8 characters. "€" uses three UTF-8 bytes
* and "😀" uses four. A pipe read boundary at a 65536-byte multiple lands inside
* a "€" sequence, because 65536 is not a multiple of three. This guarantees a
* multi-byte character split across two pipe chunks.
*/
function buildMultiByteBody(targetBytes: number): string {
const marker = "😀-start-😀";
const filler = "€".repeat(Math.ceil(targetBytes / 3));
return `${marker}${filler}${marker}`;
}
describe("duplex bridge local end-to-end harness", () => {
const harnesses: DuplexE2EHarness[] = [];
afterEach(async () => {
while (harnesses.length > 0) {
const harness = harnesses.pop();
if (harness) await harness.teardown();
}
});
const readyPredicate = (harness: DuplexE2EHarness) => () =>
harness.observedFrames.some((frame) => frame.type === "ready");
it("delivers a valid READY frame from the spawned gateway child to the attached broker", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "a READY frame from the gateway child");
const ready = harness.observedFrames.find(
(frame) => frame.type === "ready",
) as DuplexReadyFrame;
expect(ready.version).toBe(2);
expect(ready.nonce).toBe(harness.nonce);
// READY is liveness only. It carries no address data.
expect((ready as unknown as Record<string, unknown>).address).toBeUndefined();
expect((ready as unknown as Record<string, unknown>).port).toBeUndefined();
// The broker read the READY frame and stayed open with no loss.
expect(harness.broker.state).toBe("open");
expect(harness.broker.lossRecord).toBeNull();
}, 20000);
it("carries one real HTTP request through the child and the broker to the fake API unchanged", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.api.setResponder((req, res) => {
const url = new URL(req.url ?? "/", "http://127.0.0.1");
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true, echoedPath: url.pathname }));
});
const response = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(response.status).toBe(200);
await expect(response.json()).resolves.toEqual({ ok: true, echoedPath: "/api/agents/me" });
// The fake API saw one request with the real host token and the run id. The
// broker replaced the bridge token, so the fake API never saw it.
expect(harness.api.requests).toHaveLength(1);
expect(harness.api.requests[0]).toMatchObject({
method: "GET",
url: "/api/agents/me",
auth: `Bearer ${harness.hostApiToken}`,
runId: harness.runId,
});
}, 20000);
it("reassembles a large response that spans many pipe chunks", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// The body is larger than the pipe buffer, so the response frame crosses the
// child stdin pipe in many chunks. An ASCII body isolates the chunk-span
// behavior from the multi-byte behavior the next test covers.
const largeBody = "x".repeat(256 * 1024);
expect(Buffer.byteLength(largeBody, "utf8")).toBeGreaterThan(200 * 1024);
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
res.end(largeBody);
});
const response = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(response.status).toBe(200);
const received = await response.text();
// The whole body returns complete after it crossed the pipe in many chunks.
expect(received.length).toBe(largeBody.length);
expect(received).toBe(largeBody);
}, 20000);
it("decodes a multi-byte UTF-8 sequence split across chunk borders", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
const multiByteBody = buildMultiByteBody(256 * 1024);
expect(Buffer.byteLength(multiByteBody, "utf8")).toBeGreaterThan(200 * 1024);
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
res.end(multiByteBody);
});
const response = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(response.status).toBe(200);
const received = await response.text();
// A multi-byte character split across a pipe chunk border decodes to the
// same character, so the whole body returns unchanged.
expect(received.length).toBe(multiByteBody.length);
expect(received).toBe(multiByteBody);
}, 20000);
it("matches each concurrent request through one child to its own response", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.api.setResponder((req, res) => {
const url = new URL(req.url ?? "/", "http://127.0.0.1");
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ path: url.pathname }));
});
const ids = Array.from({ length: 8 }, (_unused, index) => `issue-${index}`);
const responses = await Promise.all(
ids.map((id) =>
fetch(`${harness.baseUrl}/api/issues/${id}`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
}).then(async (response) => ({ status: response.status, body: await response.json() })),
),
);
responses.forEach((result, index) => {
expect(result.status).toBe(200);
expect(result.body).toEqual({ path: `/api/issues/${ids[index]}` });
});
expect(harness.api.requests).toHaveLength(8);
}, 20000);
it("answers 409 to outstanding and 503 to new requests when the broker closes its write side", async () => {
const harness = await createHarness({ lossExitGraceMs: 3000 });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Hold the forward open, so the broker sends no response frame. The gateway
// keeps the request outstanding until the loss.
harness.setForwardMode("hang");
const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
void outstanding.catch(() => undefined);
await harness.waitFor(
() => harness.forwardedRequests.length >= 1,
"the broker to receive the forwarded request",
);
// The broker closes its write side, so the child stdin reaches a real EOF.
await harness.broker.close();
const lossResponse = await outstanding;
expect(lossResponse.status).toBe(409);
expect(lossResponse.headers.get("x-paperclip-bridge-outcome")).toBe("indeterminate");
await expect(lossResponse.json()).resolves.toEqual({ error: "outcome_indeterminate" });
const afterLoss = await fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
expect(afterLoss.status).toBe(503);
await expect(afterLoss.json()).resolves.toEqual({ error: "bridge_unavailable" });
}, 20000);
it("moves the broker to lost on a child kill, dispatches nothing more, and leaks no handle after teardown", async () => {
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Hold one forward open, so a request is in flight when the child dies.
harness.setForwardMode("hang");
const outstanding = fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
void outstanding.catch(() => undefined);
await harness.waitFor(
() => harness.forwardedRequests.length >= 1,
"the broker to receive the forwarded request",
);
const forwardedBeforeKill = harness.forwardedRequests.length;
// A real process kill closes the stdout pipe and ends the child.
await harness.killChild();
await harness.waitFor(
() => harness.broker.state === "lost",
"the broker to record the channel loss",
);
expect(harness.broker.lossRecord?.reason).toBe("channel_exit");
// The broker dispatches nothing more after the loss.
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, 100);
timer.unref?.();
});
expect(harness.forwardedRequests.length).toBe(forwardedBeforeKill);
// The outstanding fetch fails because the child died before it answered.
await expect(outstanding).rejects.toThrow();
await harness.teardown();
// Teardown left no open pipe or socket handle.
expect(harness.child.stdin.destroyed).toBe(true);
expect(harness.child.stdout.destroyed).toBe(true);
expect(harness.child.stderr.destroyed).toBe(true);
expect(harness.api.listening()).toBe(false);
expect(harness.api.openSocketCount()).toBe(0);
}, 20000);
it("fails a request with a local 502 when the host sends a malformed response chunk", async () => {
const RAW = DUPLEX_BODY_CHUNK_RAW_BYTES;
// Each case declares one malformed body_chunk a broken or hostile host could
// send. The gateway must reject it at once with a local 502. It must not grow
// its response reassembly buffer until the response timeout. `bodyByteCount`
// sets the declared body size on the response envelope. `data` is the base64
// payload of the one injected chunk.
const cases: Array<{ name: string; bodyByteCount: number; data: string; error: string }> = [
{ name: "an empty chunk", bodyByteCount: RAW, data: "", error: "duplex response body_chunk is empty" },
{
name: "an undersized non-final chunk",
bodyByteCount: RAW + 8,
data: Buffer.alloc(4, 1).toString("base64"),
error: "duplex response body_chunk has the wrong size",
},
{
name: "an oversized chunk",
bodyByteCount: RAW + 8,
data: Buffer.alloc(RAW + 4, 1).toString("base64"),
error: "duplex response body_chunk has the wrong size",
},
{
name: "an overrun past the declared size",
bodyByteCount: 10,
data: Buffer.alloc(200, 1).toString("base64"),
error: "duplex response body overruns the declared size",
},
{
name: "a non-canonical base64 chunk",
bodyByteCount: 10,
data: "AB==",
error: "duplex response body_chunk is not canonical base64",
},
];
const harness = await createHarness();
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Hold the broker forward open, so only the injected frames answer each
// request. The gateway reassembles the response body itself, so a raw
// malformed chunk on its stdin drives the reject path under test.
harness.setForwardMode("hang");
for (const testCase of cases) {
const forwardedBefore = harness.forwardedRequests.length;
const pendingFetch = fetch(`${harness.baseUrl}/api/agents/me`, {
headers: { authorization: `Bearer ${harness.bridgeToken}` },
});
void pendingFetch.catch(() => undefined);
await harness.waitFor(
() => harness.forwardedRequests.length > forwardedBefore,
`the broker to receive the forwarded request for ${testCase.name}`,
);
const id = harness.forwardedRequests[harness.forwardedRequests.length - 1].id;
// Inject the response envelope, then the one malformed body_chunk, straight
// to the gateway stdin. This bypasses the broker response encoder, so the
// gateway sees the exact bytes a broken or hostile host could send.
harness.child.stdin.write(
encodeDuplexFrame({
version: DUPLEX_FRAME_VERSION,
type: "response",
id,
status: 200,
headers: {},
bodyByteCount: testCase.bodyByteCount,
outcome: "completed",
}),
);
harness.child.stdin.write(
encodeDuplexFrame({
version: DUPLEX_FRAME_VERSION,
type: "body_chunk",
id,
seq: 0,
data: testCase.data,
}),
);
const response = await pendingFetch;
expect(response.status).toBe(502);
await expect(response.json()).resolves.toEqual({ error: testCase.error });
}
}, 20000);
it("carries a request body of exactly the API limit through the bridge to the API", async () => {
// The bridge cap is the API JSON body limit plus one byte. A body of exactly
// the API limit must pass through the bridge and reach the API. The bridge is
// a buffering backstop, not the policy limit, so it must not reject an in-spec
// body one byte early.
const apiLimitBytes = 10 * 1024 * 1024;
const bridgeCapBytes = apiLimitBytes + 1;
const harness = await createHarness({ maxBodyBytes: bridgeCapBytes });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
let receivedBodyLength = -1;
harness.api.setResponder((_req, res, body) => {
receivedBodyLength = Buffer.byteLength(body, "utf8");
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const body = "a".repeat(apiLimitBytes);
expect(Buffer.byteLength(body, "utf8")).toBe(apiLimitBytes);
const response = await fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
// The bridge did not reject the body. It reached the API unchanged.
expect(response.status).toBe(200);
expect(receivedBodyLength).toBe(apiLimitBytes);
}, 30000);
it("answers 413 from the declared length alone, before it reserves or reads any body byte", async () => {
// Regression test: the body-size overflow path threw a generic Error that
// unwound to the 502 catch. A 502 is retryable and leaked the internal
// message. The overflow must answer a clean 413 request_too_large. Assert the
// status code specifically, because the wrong 502 stood for a long time.
const maxBodyBytes = 4096;
const laterBodySize = "small".length;
// Just enough admission budget for the later small read below (its
// declared size plus the fixed in-flight frame allowance), with no
// margin. If the gateway reserved the oversize request's declared length
// before it rejected the request, this budget would stay exhausted and
// the later request would be shed with a 503 instead of admitted.
const maxInflightBodyBytes = laterBodySize + DEFAULT_MAX_DUPLEX_FRAME_BYTES;
const harness = await createHarness({ maxBodyBytes, maxInflightBodyBytes });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
let apiWasCalled = false;
harness.api.setResponder((_req, res) => {
apiWasCalled = true;
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const body = "a".repeat(maxBodyBytes + 1024);
const response = await fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
expect(response.status).toBe(413);
await expect(response.json()).resolves.toEqual({ error: "request_too_large" });
// The oversize body never reached the API, and the response leaked no
// internal message.
expect(apiWasCalled).toBe(false);
// The rejected request reserved nothing, so a small request right after
// is still admitted -- proving the gateway answered the 413 from the
// declared length header alone, before it ever reserved or read a byte.
const later = await fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: "small",
});
expect(later.status).toBe(200);
expect(apiWasCalled).toBe(true);
}, 20000);
it("reserves the worst case and answers 413 for a chunked request with no declared length that overruns the cap", async () => {
// A request with no Content-Length header -- for example a chunked
// request -- gives the gateway no declared size ahead of time. The
// gateway must fall back to the worst case (maxBodyBytes) for its
// reserve, and it must still answer a clean 413 once the body it
// actually reads exceeds the cap.
const maxBodyBytes = 4096;
const harness = await createHarness({ maxBodyBytes });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
let apiWasCalled = false;
harness.api.setResponder((_req, res) => {
apiWasCalled = true;
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const oversizeBody = "a".repeat(maxBodyBytes + 1024);
// A ReadableStream body sends with chunked transfer-encoding and no
// Content-Length header, so the gateway cannot know the real size ahead
// of time.
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(oversizeBody));
controller.close();
},
});
const response = await fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: stream,
duplex: "half",
} as RequestInit & { duplex: "half" });
expect(response.status).toBe(413);
await expect(response.json()).resolves.toEqual({ error: "request_too_large" });
expect(apiWasCalled).toBe(false);
// The rejected read released its reserve, so a later request is admitted
// again and reaches the API.
const later = await fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: "small",
});
expect(later.status).toBe(200);
}, 20000);
it("charges a bodyless DELETE the zero-length reserve, not the worst-case maxBodyBytes charge", async () => {
// Regression coverage for PAP-5046. A DELETE with no body sends neither a
// Content-Length header nor a Transfer-Encoding header. Before the fix,
// declaredBodyBytes returned null for that shape and the gateway charged
// the worst-case maxBodyBytes reserve, the same charge as an unknown-size
// chunked body. This pins maxInflightBodyBytes to fit the zero-length
// charge (0 + DEFAULT_MAX_DUPLEX_FRAME_BYTES) but not the worst-case
// charge (maxBodyBytes + DEFAULT_MAX_DUPLEX_FRAME_BYTES), so only the
// fixed classification admits the request.
const maxBodyBytes = 4096;
const maxInflightBodyBytes = DEFAULT_MAX_DUPLEX_FRAME_BYTES;
const harness = await createHarness({ maxBodyBytes, maxInflightBodyBytes });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
let apiWasCalled = false;
harness.api.setResponder((_req, res) => {
apiWasCalled = true;
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
// /api/routine-triggers/{id} is the one DELETE route the host allowlist
// admits; a DELETE elsewhere gets a 403 before it reaches the API,
// regardless of this fix, so this path isolates the admission charge.
const response = await fetch(`${harness.baseUrl}/api/routine-triggers/t-1`, {
method: "DELETE",
headers: { authorization: `Bearer ${harness.bridgeToken}`, "content-type": "application/json" },
});
expect(response.status).not.toBe(503);
expect(apiWasCalled).toBe(true);
}, 20000);
it("admits many small concurrent requests that the old flat 10 MiB charge would have shed with 503", async () => {
// Production constants. Under the old flat charge every active read
// reserved maxBodyBytes (about 10 MiB) regardless of the real body size,
// so the 64 MiB inflight bound admitted at most
// floor(64 MiB / (10 MiB + 1)) = 6 concurrent reads. Charging the
// declared body size instead means a control-plane-sized body reserves
// only its own few hundred bytes plus the fixed frame allowance, so the
// same bound admits far more than 6 concurrent small reads at once.
const maxBodyBytes = 10 * 1024 * 1024 + 1;
const maxInflightBodyBytes = 64 * 1024 * 1024;
const concurrency = 20;
const harness = await createHarness({ maxBodyBytes, maxInflightBodyBytes });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const body = JSON.stringify({ body: "a small status comment" });
const post = () =>
fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
const responses = await Promise.all(Array.from({ length: concurrency }, () => post()));
expect(responses.map((response) => response.status)).toEqual(
Array.from({ length: concurrency }, () => 200),
);
}, 20000);
it("sheds a concurrent body read past the admission bound, then releases the reserve", async () => {
// The gateway reserves the declared body size plus the fixed in-flight
// frame allowance for each active read against the local admission
// bound. With the bound set to exactly one such reserve, the first read
// is admitted and the second read is shed with a 503. The gateway
// releases the reserve on the response timeout, so a later request is
// admitted again.
const maxBodyBytes = 4096;
const bodySize = 1024;
const reservePerRead = bodySize + DEFAULT_MAX_DUPLEX_FRAME_BYTES;
const harness = await createHarness({
maxBodyBytes,
// The bound admits exactly one concurrent body read at bodySize.
maxInflightBodyBytes: reservePerRead,
// A short response deadline releases the first (hung) read fast.
responseTimeoutMs: 700,
});
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// The first admitted request hangs on the forward until it times out. The
// second request cannot reserve, so the gateway sheds it with a 503.
harness.setForwardMode("hang");
const body = "a".repeat(bodySize);
const post = () =>
fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
const [first, second] = await Promise.all([post(), post()]);
const statuses = [first.status, second.status].sort((a, b) => a - b);
// One request was admitted (it later times out with a 502). One request was
// shed with a 503 before it could buffer a second body.
expect(statuses).toEqual([502, 503]);
const shed = first.status === 503 ? first : second;
await expect(shed.json()).resolves.toEqual({ error: "bridge_busy" });
// The admitted read timed out and released its reserve. A later request is
// admitted again and reaches the API.
harness.setForwardMode("proxy");
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const later = await post();
expect(later.status).toBe(200);
}, 20000);
it("reserves twice the body cap for an unknown-length read, so three cap-sized chunked reads exhaust the bound", async () => {
// Regression coverage for PAP-5049 (Stage-2 round 7). readBody holds a
// chunk list and the Buffer.concat result together for one instant when
// the declared length is unknown -- see its comment -- so an
// unknown-length read's true retained peak is up to two times
// maxBodyBytes, not one time. The reserve charge must match that peak:
// 2 * maxBodyBytes plus the fixed frame allowance (the 2x factor here
// matches DUPLEX_GATEWAY_UNKNOWN_LENGTH_BODY_RESERVE_FACTOR in the
// gateway source). This test sets the in-flight bound to admit exactly
// three such reads at once, holds the host forward so the reserves
// overlap, and proves a fourth read is shed with a 503 -- then proves a
// later read is admitted once a terminal releases a reserve.
//
// maxBodyBytes must be large relative to the fixed frame allowance for
// the 2x factor to change the admitted count: a body far smaller than
// the frame allowance would admit the same count either way, and the
// test would not tell the two charges apart.
const maxBodyBytes = 2 * 1024 * 1024;
const reservePerRead = 2 * maxBodyBytes + DEFAULT_MAX_DUPLEX_FRAME_BYTES;
const admitCount = 3;
const harness = await createHarness({
maxBodyBytes,
maxInflightBodyBytes: admitCount * reservePerRead,
// A short response deadline releases each hung read fast.
responseTimeoutMs: 700,
});
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Each admitted read hangs on the forward until it times out, so all
// admitted reads hold their reserve at once and the next read cannot
// reserve.
harness.setForwardMode("hang");
const chunkedBody = "a".repeat(maxBodyBytes);
const postChunked = () => {
// A ReadableStream body sends with chunked transfer-encoding and no
// Content-Length header, so the gateway cannot know the real size
// ahead of time.
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(chunkedBody));
controller.close();
},
});
return fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: stream,
duplex: "half",
} as RequestInit & { duplex: "half" });
};
const responses = await Promise.all(
Array.from({ length: admitCount + 1 }, () => postChunked()),
);
const statuses = responses.map((response) => response.status);
// Three reads were admitted (each later times out with a 502). One read
// past the model is shed with a 503 before it buffers a body.
const shedCount = statuses.filter((status) => status === 503).length;
const admittedTimedOut = statuses.filter((status) => status === 502).length;
expect(shedCount).toBe(1);
expect(admittedTimedOut).toBe(admitCount);
const shed = responses.find((response) => response.status === 503);
await expect(shed?.json()).resolves.toEqual({ error: "bridge_busy" });
// The admitted reads timed out and released their reserve. A later
// chunked request is admitted again and reaches the API.
harness.setForwardMode("proxy");
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const later = await postChunked();
expect(later.status).toBe(200);
}, 30000);
it("forwards a cap-sized malformed-UTF-8 body byte-for-byte and emits no expanded frame", async () => {
// Regression test for the admission-bypass block. The gateway used to decode
// the raw body to a UTF-8 string, then re-encode it. Each invalid byte
// became the three-byte U+FFFD replacement, so a cap-sized malformed body
// expanded to about three times the bytes that admission charged. The gateway
// now preserves the raw bytes, so the body rides base64 frames unchanged and
// no frame carries more than the charged bytes.
const maxBodyBytes = 700 * 1024;
const harness = await createHarness({ maxBodyBytes });
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
// 0xFF is never a valid UTF-8 byte. A cap-sized body of 0xFF bytes is the
// worst case for the old expansion path.
const malformed = Buffer.alloc(maxBodyBytes, 0xff);
const response = await fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: new Uint8Array(malformed),
});
expect(response.status).toBe(200);
// The observed frames are the exact bytes the gateway sent to the host. The
// envelope reports the raw byte count, not an expanded count.
const requestFrame = harness.observedFrames.find(
(frame): frame is DuplexRequestFrame => frame.type === "request",
);
expect(requestFrame?.bodyByteCount).toBe(maxBodyBytes);
// The body_chunk frames decode to the exact source bytes. No frame carried an
// expanded body, so no body larger than the charged bytes reached the host.
const bodyChunks = harness.observedFrames.filter(
(frame): frame is DuplexBodyChunkFrame => frame.type === "body_chunk",
);
const reassembled = Buffer.concat(
bodyChunks.map((frame) => Buffer.from(frame.data, "base64")),
);
expect(reassembled.length).toBe(maxBodyBytes);
expect(reassembled.equals(malformed)).toBe(true);
// The route stays usable after the malformed body.
const later = await fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: JSON.stringify({ ok: true }),
});
expect(later.status).toBe(200);
}, 30000);
it("caps concurrent body reads at the retained-byte model, then releases the reserve", async () => {
// Production-size accounting. The gateway charges each active read the
// declared body size plus the fixed in-flight frame allowance, not the
// flat body cap, so the bound admits
// floor(inflightBound / (declaredBytes + frameAllowance)) reads. The
// declared body here is well under the cap, so this model admits far
// more than the old flat-charge model's 6 -- but the test still keeps
// the count under the gateway's separate maxQueueDepth bound (64), so
// that unrelated queue-depth gate does not also shed a request and
// confound the assertions below.
const maxBodyBytes = 10 * 1024 * 1024 + 1;
const maxInflightBodyBytes = 64 * 1024 * 1024;
const bodySize = 1024 * 1024;
const reservePerRead = bodySize + DEFAULT_MAX_DUPLEX_FRAME_BYTES;
const admitCount = Math.floor(maxInflightBodyBytes / reservePerRead);
expect(admitCount).toBe(32);
const harness = await createHarness({
maxBodyBytes,
maxInflightBodyBytes,
// A short response deadline releases each hung read fast.
responseTimeoutMs: 800,
});
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Each admitted read hangs on the forward until it times out, so all admitted
// reads hold their reserve at once and the next read cannot reserve.
harness.setForwardMode("hang");
const body = "a".repeat(bodySize);
const post = () =>
fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
const responses = await Promise.all(
Array.from({ length: admitCount + 1 }, () => post()),
);
const statuses = responses.map((response) => response.status);
// The model admits exactly admitCount reads. Each admitted read times out
// with a 502. The one read past the model is shed with a 503 before it
// buffers a body.
const shedCount = statuses.filter((status) => status === 503).length;
const admittedTimedOut = statuses.filter((status) => status === 502).length;
expect(shedCount).toBe(1);
expect(admittedTimedOut).toBe(admitCount);
const shed = responses.find((response) => response.status === 503);
await expect(shed?.json()).resolves.toEqual({ error: "bridge_busy" });
// The admitted reads released their reserve on timeout. A later request is
// admitted again and reaches the API.
harness.setForwardMode("proxy");
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const later = await post();
expect(later.status).toBe(200);
}, 30000);
it("bounds the outbound writer to about one frame per active send while the pipe backpressures", async () => {
// Regression test for the backpressure block. The gateway used to call
// stdout.write() for every frame in a tight loop and ignore the return
// value, so a slow reader let one admitted read queue its whole body's
// worth of encoded frames. Pause the pipe read side, so every later write
// meets a full pipe, then post several concurrent cap-sized bodies at
// once and prove the writer never reports more than about one bounded
// frame stuck behind the full pipe.
const maxBodyBytes = 4 * 1024 * 1024;
const concurrency = 4;
const harness = await createHarness({
maxBodyBytes,
// Each concurrent read charges its declared body size plus the fixed
// in-flight frame allowance, so the bound must admit all four at once.
maxInflightBodyBytes: (maxBodyBytes + DEFAULT_MAX_DUPLEX_FRAME_BYTES) * concurrency,
responseTimeoutMs: 20000,
});
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Stop reading the child's stdout, so the OS pipe fills and the gateway's
// writes meet real backpressure, the same as a slow host reader would
// cause in production.
harness.child.stdout.pause();
const body = "a".repeat(maxBodyBytes);
const posts = Array.from({ length: concurrency }, (_unused, index) =>
fetch(`${harness.baseUrl}/api/issues/issue-${index}/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
}).catch(() => undefined),
);
// Give the gateway time to read every body and drive its sends into
// backpressure.
await new Promise((resolve) => setTimeout(resolve, 1500));
const backpressureLines = harness
.stderr()
.split("\n")
.filter((line) => line.includes("stdout frame write backpressured"));
expect(backpressureLines.length).toBeGreaterThan(0);
const reportedLengths = backpressureLines.map((line) => {
const match = /writableLength=(\d+)/.exec(line);
if (!match) throw new Error("backpressure diagnostic is missing writableLength: " + line);
return Number(match[1]);
});
// One buffered frame is at most DEFAULT_MAX_DUPLEX_FRAME_BYTES (1 MB). An
// unbounded queue would report a writableLength close to one whole body's
// worth of encoded frames -- several megabytes for a 4 MiB body. A
// generous multiple of one frame proves the writer never queued anywhere
// near that, regardless of how long the pipe stays full.
const oneFrameBound = DEFAULT_MAX_DUPLEX_FRAME_BYTES * 2;
for (const length of reportedLengths) {
expect(length).toBeLessThan(oneFrameBound);
}
// Resume so the harness can read the outstanding frames and tear down
// cleanly.
harness.child.stdout.resume();
await Promise.allSettled(posts);
}, 20000);
it("keeps one request's frames in sequence while a concurrent request interleaves its own", async () => {
const harness = await createHarness({
maxBodyBytes: 2 * 1024 * 1024,
maxInflightBodyBytes: 8 * 1024 * 1024,
});
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
// Each body is large enough to split into several body_chunk frames, so a
// correct sequence is not a coincidence of a one-frame body.
const bodyA = "a".repeat(1_500_000);
const bodyB = "b".repeat(1_500_000);
const post = (body: string) =>
fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
const [responseA, responseB] = await Promise.all([post(bodyA), post(bodyB)]);
expect(responseA.status).toBe(200);
expect(responseB.status).toBe(200);
const requestFrames = harness.observedFrames.filter(
(frame): frame is DuplexRequestFrame => frame.type === "request",
);
expect(requestFrames).toHaveLength(2);
for (const requestFrame of requestFrames) {
const chunkSeqs = harness.observedFrames
.filter(
(frame): frame is DuplexBodyChunkFrame =>
frame.type === "body_chunk" && frame.id === requestFrame.id,
)
.map((frame) => frame.seq);
expect(chunkSeqs.length).toBeGreaterThan(1);
// This request's own chunks arrive in increasing sequence order, even
// though the other request's frames may interleave between them.
expect(chunkSeqs).toEqual(Array.from({ length: chunkSeqs.length }, (_unused, index) => index));
}
}, 20000);
it("holds the admission reserve while a send is stalled behind a full pipe, then releases it once the send drains", async () => {
const maxBodyBytes = 2 * 1024 * 1024;
const harness = await createHarness({
maxBodyBytes,
// Admits exactly one concurrent body read at maxBodyBytes: the declared
// body size plus the fixed in-flight frame allowance.
maxInflightBodyBytes: maxBodyBytes + DEFAULT_MAX_DUPLEX_FRAME_BYTES,
responseTimeoutMs: 15000,
});
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
harness.setForwardMode("hang");
// Stop reading the child's stdout, so the first request's send stalls
// mid-flight, well before it ever reaches the host.
harness.child.stdout.pause();
const body = "a".repeat(maxBodyBytes);
const first = fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
void first.catch(() => undefined);
// Give the gateway time to read the body and drive its send into
// backpressure.
await new Promise((resolve) => setTimeout(resolve, 500));
// The reserve is still held: a second request cannot reserve, so the
// gateway sheds it with a clean 503.
const shed = await fetch(`${harness.baseUrl}/api/issues/issue-2/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: "small",
});
expect(shed.status).toBe(503);
await expect(shed.json()).resolves.toEqual({ error: "bridge_busy" });
// Resume the pipe and let the host answer. The stalled send drains and
// finishes, and the first request's response settles.
harness.child.stdout.resume();
harness.setForwardMode("proxy");
harness.api.setResponder((_req, res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
const firstResponse = await first;
expect(firstResponse.status).toBe(200);
// The reserve released once the send finished. A later request is
// admitted again.
const later = await fetch(`${harness.baseUrl}/api/issues/issue-3/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: "small",
});
expect(later.status).toBe(200);
}, 20000);
it("ends a stalled send when the outbound pipe breaks, and releases the reserve for a later request", async () => {
const maxBodyBytes = 2 * 1024 * 1024;
const harness = await createHarness({
maxBodyBytes,
// Admits exactly one concurrent body read at maxBodyBytes: the declared
// body size plus the fixed in-flight frame allowance.
maxInflightBodyBytes: maxBodyBytes + DEFAULT_MAX_DUPLEX_FRAME_BYTES,
responseTimeoutMs: 1200,
});
harnesses.push(harness);
await harness.waitFor(readyPredicate(harness), "the gateway to become ready");
// Stop reading the child's stdout, so the first request's send is
// genuinely in flight -- stalled behind a full pipe -- when the pipe
// breaks.
harness.child.stdout.pause();
const body = "a".repeat(maxBodyBytes);
const first = fetch(`${harness.baseUrl}/api/issues/issue-1/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body,
});
void first.catch(() => undefined);
// Give the gateway time to read the body and start writing frames into
// the full pipe.
await new Promise((resolve) => setTimeout(resolve, 400));
// Confirm the reserve is held before the pipe breaks: a second request
// cannot reserve, so the gateway sheds it with a clean 503.
const shedBeforeBreak = await fetch(`${harness.baseUrl}/api/issues/issue-2/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: "small",
});
expect(shedBeforeBreak.status).toBe(503);
// Break the pipe. The stalled write meets a stream error, which ends the
// wait for drain -- the only other end condition the writer accepts.
harness.child.stdout.destroy();
const firstResponse = await first;
// No response frame can ever arrive over a broken pipe. The gateway's own
// wait budget ends the request with its local 502 timeout.
expect(firstResponse.status).toBe(502);
// The reserve released once the failed send and the timed-out response
// both finished. A later request is admitted again -- it is not shed with
// 503, even though it can never complete over the broken pipe either.
const later = await fetch(`${harness.baseUrl}/api/issues/issue-3/comments`, {
method: "POST",
headers: {
authorization: `Bearer ${harness.bridgeToken}`,
"content-type": "application/json",
},
body: "small",
});
expect(later.status).toBe(502);
}, 20000);
});