From 141b815294fa0dbb5bf15e3d52023f8db011bb4a Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Sat, 22 Aug 2026 16:32:18 -0700 Subject: [PATCH 1/6] fix(adapter-utils): enforce the duplex frame size bound on encode in both codec copies (#11983) ## Thinking Path > - Paperclip uses adapter utilities to move bounded messages between agent processes. > - The duplex frame codec encodes and decodes these messages. > - The decoder rejects frames above the documented byte limit. > - The encoder did not apply the same limit before it sent a frame. > - This mismatch let a sender write a frame that the peer rejected after transmission. > - This pull request applies the same byte limit to both codec copies and keeps the broker channel open. > - The benefit is a local error with stable request telemetry instead of a channel loss. ## Linked Issues or Issue Description No public GitHub issue exists for this change. The problem follows the bug report fields below. **What happened?** The duplex encoder could write a frame larger than `DEFAULT_MAX_DUPLEX_FRAME_BYTES`. The peer decoder then rejected the frame after transmission. In the WebSocket 1009 case, this closed the channel and reported a process exit. **Expected behavior** The encoder should reject an oversized frame before it writes bytes. The gateway should return HTTP 413. The broker should return a bounded terminal response and keep other requests active. **Steps to reproduce** 1. Encode a duplex frame above `DEFAULT_MAX_DUPLEX_FRAME_BYTES`. 2. Send the frame through the gateway or broker. 3. Observe that the old path writes the frame or drops the channel after peer rejection. **Paperclip version or commit** Reproduced from the `master` development line before this change. **Deployment mode** Local dev (`pnpm dev`). ## What Changed - Add `encodeDuplexFrameChecked` to the host and embedded gateway codecs. - Measure encoded JSON bytes without the trailing newline. - Return a typed `frame_too_large` result without throwing. - Return HTTP 413 for oversized gateway requests without writing a frame. - Share one frame bound between broker decode and encode checks. - Return a bounded, non-retryable terminal response for oversized broker responses. - Add encode vectors to the shared wire-compatibility fixture. ## Verification - Run `pnpm --filter @paperclipai/adapter-utils typecheck`. - Run `npx vitest run packages/adapter-utils/src/duplex-frame-codec.test.ts packages/adapter-utils/src/duplex-bridge-broker.test.ts packages/adapter-utils/src/execution-target-sandbox.test.ts`. - Confirm the oversized-response broker test keeps the channel open and serves the other in-flight request. - Confirm the gateway test returns HTTP 413 and keeps the channel open. ## Risks The encoder now rejects oversized frames before transmission. This changes an unsafe write into a typed local error. The broker and gateway keep existing frame limits and affect only oversized frames. ## Model Used OpenAI Codex, GPT-5, tool use and code execution. The model assisted with review and repository operations. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/duplex-bridge-broker.test.ts | 226 ++++++++++++++++++ .../adapter-utils/src/duplex-bridge-broker.ts | 120 +++++++++- .../src/duplex-frame-codec.test.ts | 80 +++++++ .../adapter-utils/src/duplex-frame-codec.ts | 33 +++ .../src/duplex-frame-vectors.json | 50 ++++ .../src/execution-target-sandbox.test.ts | 96 +++++++- .../src/sandbox-callback-bridge.ts | 28 ++- 7 files changed, 616 insertions(+), 17 deletions(-) diff --git a/packages/adapter-utils/src/duplex-bridge-broker.test.ts b/packages/adapter-utils/src/duplex-bridge-broker.test.ts index d254742a269..fccddb3b820 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.test.ts @@ -461,6 +461,232 @@ describe("duplex bridge broker request limits", () => { expect(broker.state).toBe("open"); }); + it("substitutes a bounded terminal response for an oversized response and keeps the channel open", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + // A small frame bound makes a legitimately large response body exceed the + // bound, so the test exercises the encode guard without a megabyte body. + maxFrameBytes: 1000, + }); + brokers.push(broker); + broker.start(); + + // Two requests are in flight at the same time. One resolves with a normal + // body; the other resolves with a body that pushes the response frame over + // the bound. The host produced both results, so both requests reached the + // host. + harness.feed(requestFrame("small-1", "POST")); + harness.feed(requestFrame("big-1", "POST")); + + const smallForward = harness.forwards.find((entry) => entry.id === "small-1" && !entry.settled); + if (!smallForward) throw new Error("The broker did not forward the small request."); + smallForward.settled = true; + smallForward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ok: true }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + const bigForward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); + if (!bigForward) throw new Error("The broker did not forward the big request."); + bigForward.settled = true; + bigForward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ data: "y".repeat(2000) }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The other in-flight request delivered its full response, unaffected by the + // oversized one. + expect(harness.responses).toHaveLength(2); + const delivered = harness.responses.find((entry) => entry.id === "small-1")!; + expect(delivered.status).toBe(200); + expect(delivered.outcome).toBe("completed"); + expect(JSON.parse(delivered.body)).toEqual({ ok: true }); + + // The oversized response became one bounded terminal response marked + // indeterminate and non-retryable, so a caller does not retry a possible + // mutation. The broker did not write the oversized frame. + const terminal = harness.responses.find((entry) => entry.id === "big-1")!; + expect(terminal.status).toBe(502); + expect(terminal.outcome).toBe("indeterminate"); + expect(terminal.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); + expect(JSON.parse(terminal.body)).toEqual({ + error: "upstream response too large to deliver", + outcome: "indeterminate", + retryable: false, + }); + + // The oversized body never crossed the wire. The broker did not truncate it or + // pass it through. + const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); + expect(writtenText).not.toContain("yyyy"); + + // The broker recorded the oversized request as an error outcome, never a loss. + // The channel stays open and the run does not fail. + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(1); + expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + + it("sends a minimal terminal response when the frame bound rejects the full replacement", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + // The bound accepts a compact request (130 bytes) and the minimal terminal + // response (106 bytes), but rejects the full replacement response (288 bytes). + // This proves the broker never drops the channel or strands the request when + // the configured bound is smaller than the full replacement response. + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + maxFrameBytes: 200, + }); + brokers.push(broker); + broker.start(); + + // A compact request frame fits the bound, so the broker forwards it. + harness.feed({ + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "big-1", + method: "POST", + path: "/api/issues/big-1", + query: "", + headers: {}, + body: JSON.stringify({ go: 1 }), + }); + + const forward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); + if (!forward) throw new Error("The broker did not forward the request."); + forward.settled = true; + // The host produced a large response body that pushes the real response frame + // and the full replacement frame over the bound. + forward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ data: "y".repeat(2000) }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The broker sent one minimal terminal response, not the oversized real + // response and not the full replacement. The minimal response carries the + // indeterminate outcome, so the gateway maps it to a terminal 409. + expect(harness.responses).toHaveLength(1); + const terminal = harness.responses[0]!; + expect(terminal.id).toBe("big-1"); + expect(terminal.status).toBe(502); + expect(terminal.outcome).toBe("indeterminate"); + expect(terminal.headers).toEqual({}); + expect(terminal.body).toBe(""); + + // The oversized body never crossed the wire. + const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); + expect(writtenText).not.toContain("yyyy"); + + // The broker recorded the request as an error outcome, never a loss. The + // channel stays open and the run does not fail. + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(1); + expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + + it("keeps the channel open and logs when the bound rejects even the minimal terminal response", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + const logs: string[] = []; + // The bound accepts a tiny request (99 bytes) but rejects even the minimal + // terminal response (102 bytes). The broker cannot send any frame for this + // request, so it must keep the channel open and log a clear local error. + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + maxFrameBytes: 100, + logger: (message) => logs.push(message), + }); + brokers.push(broker); + broker.start(); + + harness.feed({ + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "x", + method: "GET", + path: "/", + query: "", + headers: {}, + body: "", + }); + + const forward = harness.forwards.find((entry) => entry.id === "x" && !entry.settled); + if (!forward) throw new Error("The broker did not forward the request."); + forward.settled = true; + forward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ data: "y".repeat(200) }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The broker sent no response frame, because no frame fits the bound. + expect(harness.responses).toHaveLength(0); + // The broker recorded the request as an error outcome, never a loss. + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(1); + // The broker logged a clear local error and kept the channel open. + expect(logs.some((line) => line.includes("could not deliver a terminal response"))).toBe(true); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + + it("completes a request within the bound and delivers the real response", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + maxFrameBytes: 1000, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("ok-1", "POST")); + harness.resolveForward("ok-1", JSON.stringify({ ok: true })); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The broker delivered the real response promptly with the completed outcome. + expect(harness.responses).toHaveLength(1); + const delivered = harness.responses[0]!; + expect(delivered.id).toBe("ok-1"); + expect(delivered.status).toBe(200); + expect(delivered.outcome).toBe("completed"); + expect(JSON.parse(delivered.body)).toEqual({ ok: true }); + + // The delivered response records an `ok` span, never an error, and never a loss. + const okSpans = capture.spans.filter((span) => span.dimensions.outcome === "ok"); + expect(okSpans).toHaveLength(1); + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(0); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + it("maps a forward rejection for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); const broker = createDuplexBridgeBroker({ diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts index 2cba4097660..43d7b524f37 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.ts @@ -43,10 +43,12 @@ import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; import { + DEFAULT_MAX_DUPLEX_FRAME_BYTES, DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, DUPLEX_FRAME_VERSION, DuplexFrameDecoder, encodeDuplexFrame, + encodeDuplexFrameChecked, type DuplexFrame, type DuplexRequestFrame, type DuplexResponseFrame, @@ -376,9 +378,12 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS; const closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS; const now = options.now ?? (() => Date.now()); - const decoder = new DuplexFrameDecoder( - options.maxFrameBytes ? { maxFrameBytes: options.maxFrameBytes } : undefined, - ); + // The one frame size bound the broker enforces on both sides. The decoder + // rejects an inbound frame over this bound, and the encode guard refuses to + // write an outbound frame over it. Encode and decode share one value, so a + // frame the broker writes always decodes on the peer. + const maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES; + const decoder = new DuplexFrameDecoder({ maxFrameBytes }); let state: DuplexBrokerState = "opening"; let stopped = false; @@ -491,9 +496,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr options.telemetry?.recordLoss(lossClass, BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"); }; - const writeFrame = (frame: DuplexFrame): boolean => { + const writeLine = (line: string): boolean => { try { - channel.write(encodeDuplexFrame(frame)); + channel.write(line); return true; } catch (error) { recordLoss("stream_failure", errorMessage(error)); @@ -501,6 +506,82 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } }; + // The result of one bounded send. `sent` means the frame went out; `too_large` + // means the encoded frame exceeds the bound and the broker wrote nothing; + // `lost` means the write failed and the broker recorded the channel loss. + type SendResult = "sent" | "too_large" | "lost"; + + const trySendFrame = (frame: DuplexFrame): SendResult => { + // Enforce the frame size bound on every broker write. Report a size rejection + // as `too_large` without a channel loss; a broker-built frame over the bound + // is a defect, not a transport failure. A caller decides how to answer the + // request, so no size rejection ever drops the channel. + const encoded = encodeDuplexFrameChecked(frame, maxFrameBytes); + if (!encoded.ok) return "too_large"; + return writeLine(encoded.line) ? "sent" : "lost"; + }; + + const writeFrame = (frame: DuplexFrame): boolean => { + // A control frame and the bounded terminal responses are always small, so the + // size guard is a no-op for them. Drop an oversized frame here without a + // channel loss and report the drop to the caller. + const result = trySendFrame(frame); + if (result === "too_large") { + options.logger?.(`Duplex broker dropped an oversized ${frame.type} frame.`); + return false; + } + return result === "sent"; + }; + + const sendTerminalIndeterminate = (id: string): void => { + // Answer one request the broker cannot deliver with the real result. The + // request reached the host and may have changed state, so the response is + // non-retryable and carries the `indeterminate` outcome. The broker tries the + // full replacement first. When the frame bound rejects even the full + // replacement, the broker sends a minimal replacement that carries only the + // `indeterminate` outcome, so the gateway still ends the request and never + // waits for its full wait budget. When the bound rejects even the minimal + // replacement, the broker logs a clear local error and keeps the channel + // open; it records no channel loss. + const fullReplacement: DuplexResponseFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: 502, + headers: { + "content-type": "application/json", + "x-paperclip-bridge-outcome": "indeterminate", + }, + body: JSON.stringify({ + error: "upstream response too large to deliver", + outcome: "indeterminate", + retryable: false, + }), + outcome: "indeterminate", + }; + if (trySendFrame(fullReplacement) !== "too_large") return; + // The bound rejects the full replacement. Send a minimal terminal response + // with empty headers and an empty body. The `indeterminate` outcome still + // rides the frame, so the gateway maps the request to a terminal 409. + const minimalReplacement: DuplexResponseFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: 502, + headers: {}, + body: "", + outcome: "indeterminate", + }; + if (trySendFrame(minimalReplacement) !== "too_large") return; + // The bound rejects even the minimal terminal response. The broker cannot + // deliver any frame for this request within the bound. Log a clear local + // error and keep the channel open for every other request. The gateway ends + // its own outstanding request on its wait budget. + options.logger?.( + `Duplex broker could not deliver a terminal response within the ${maxFrameBytes}-byte frame bound.`, + ); + }; + const respond = ( id: string, result: DuplexBrokerForwardResult, @@ -516,12 +597,6 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // Do not write on a lost or closed channel. The gateway answers its own // outstanding request on loss, so a late write would go to a dead channel. if (state !== "open") return; - // Record the request span for the delivered request. The span carries the - // latency and the outcome only; no route, query, body, or token rides it. - options.telemetry?.recordRequest({ - latencyMs: now() - entry.dispatchStartMs, - outcome: telemetryOutcome, - }); const frame: DuplexResponseFrame = { version: DUPLEX_FRAME_VERSION, type: "response", @@ -531,7 +606,28 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr body: result.body ?? "", outcome, }; - writeFrame(frame); + const encoded = encodeDuplexFrameChecked(frame, maxFrameBytes); + if (!encoded.ok) { + // The host produced a result, but the response frame exceeds the size + // bound. Do not write the oversized frame and do not record a channel loss. + // The request reached the host and may have changed state, so the caller + // must not retry a possible mutation. Send a bounded, non-retryable terminal + // response marked `indeterminate` in its place. Record the request outcome + // as an error, never a loss. The channel stays open for every other request. + options.telemetry?.recordRequest({ + latencyMs: now() - entry.dispatchStartMs, + outcome: "error", + }); + sendTerminalIndeterminate(id); + return; + } + // Record the request span for the delivered request. The span carries the + // latency and the outcome only; no route, query, body, or token rides it. + options.telemetry?.recordRequest({ + latencyMs: now() - entry.dispatchStartMs, + outcome: telemetryOutcome, + }); + writeLine(encoded.line); }; const respondSaturated = (id: string, retryable: boolean): void => { diff --git a/packages/adapter-utils/src/duplex-frame-codec.test.ts b/packages/adapter-utils/src/duplex-frame-codec.test.ts index e9c00b0b75d..42ec35a4dbc 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.test.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.test.ts @@ -8,6 +8,7 @@ import { DuplexFrameDecoder, decodeDuplexLine, encodeDuplexFrame, + encodeDuplexFrameChecked, type DuplexDecodeResult, type DuplexFrame, } from "./duplex-frame-codec.js"; @@ -33,10 +34,20 @@ interface Vector { expected: ExpectedResult[]; } +// One encode-bound vector: a frame, the size limit, and the expected checked +// encode result. An ok result must also decode back to the same frame. +interface EncodeVector { + name: string; + maxFrameBytes: number; + frame: DuplexFrame; + expected: { ok: true } | { ok: false; error: string }; +} + interface Fixture { frameVersion: number; defaultMaxFrameBytes: number; vectors: Vector[]; + encodeVectors: EncodeVector[]; } const fixturePath = fileURLToPath( @@ -310,3 +321,72 @@ describe("request id byte bound", () => { if (!result.ok) expect(result.error.code).toBe("id_too_large"); }); }); + +describe("size-checked encode", () => { + it("runs every shared encode vector and matches the expected result", () => { + // Every codec copy runs the same encode vectors. This copy proves it enforces + // the same bound the embedded gateway copy enforces. + for (const vector of fixture.encodeVectors) { + const result = encodeDuplexFrameChecked(vector.frame, vector.maxFrameBytes); + expect(result.ok).toBe(vector.expected.ok); + if (result.ok) { + // An ok line ends with one newline and decodes back to the same frame, so + // the encode guard never truncates or passes a bad frame through. + expect(result.line.endsWith("\n")).toBe(true); + expect(result.line.slice(0, -1)).not.toContain("\n"); + const decoded = decodeDuplexLine(result.line.slice(0, -1)); + expect(decoded.ok).toBe(true); + if (decoded.ok) expect(decoded.frame).toEqual(vector.frame); + } else if (!vector.expected.ok) { + expect(result.error.code).toBe(vector.expected.error); + } + } + }); + + it("rejects an over-bound frame with a typed outcome and never throws", () => { + const frame: DuplexFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id: "big", + status: 200, + headers: {}, + body: "x".repeat(2_000), + outcome: "completed", + }; + expect(() => encodeDuplexFrameChecked(frame, 1_000)).not.toThrow(); + const result = encodeDuplexFrameChecked(frame, 1_000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.code).toBe("frame_too_large"); + }); + + it("measures the bound in bytes without the trailing newline, so the boundary is inclusive", () => { + // Build a frame whose encoded JSON is exactly N bytes. The guard measures the + // JSON without the newline, so N bytes encodes and N+1 bytes fails. This is the + // same boundary the decoder applies, so encode and decode agree exactly. + const base: DuplexFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id: "boundary", + status: 200, + headers: {}, + body: "", + outcome: "completed", + }; + const baseBytes = Buffer.byteLength(JSON.stringify(base), "utf8"); + const limit = baseBytes + 10; + const atLimit: DuplexFrame = { ...base, body: "x".repeat(10) }; + expect(Buffer.byteLength(JSON.stringify(atLimit), "utf8")).toBe(limit); + const okResult = encodeDuplexFrameChecked(atLimit, limit); + expect(okResult.ok).toBe(true); + const overResult = encodeDuplexFrameChecked({ ...base, body: "x".repeat(11) }, limit); + expect(overResult.ok).toBe(false); + if (!overResult.ok) expect(overResult.error.code).toBe("frame_too_large"); + }); + + it("defaults the bound to the default max frame bytes", () => { + const frame: DuplexFrame = { version: DUPLEX_FRAME_VERSION, type: "heartbeat" }; + const result = encodeDuplexFrameChecked(frame); + expect(result.ok).toBe(true); + if (result.ok) expect(result.line).toBe(encodeDuplexFrame(frame)); + }); +}); diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index 2efdb384a23..20cd814506a 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -147,6 +147,15 @@ export type DuplexDecodeResult = | { ok: true; frame: DuplexFrame } | { ok: false; error: DuplexProtocolError }; +/** + * One size-checked encode result: one line, or a `frame_too_large` error. The + * shape mirrors {@link DuplexDecodeResult}, so the encode side reports the + * over-limit case as a typed outcome, not an exception. + */ +export type DuplexEncodeResult = + | { ok: true; line: string } + | { ok: false; error: { code: "frame_too_large"; message: string } }; + const RESPONSE_OUTCOMES: ReadonlySet = new Set([ "completed", "indeterminate", @@ -187,6 +196,30 @@ export function encodeDuplexFrame(frame: DuplexFrame): string { return `${JSON.stringify(frame)}\n`; } +/** + * Encode one frame to a single line and enforce the maximum frame size. The + * function measures the encoded JSON in bytes, without the trailing newline, so + * it matches the decoder bound exactly: a line the decoder accepts encodes, and a + * line the decoder rejects returns a `frame_too_large` result. The function never + * throws; it reports the over-limit case as a typed outcome. + * + * The size bound follows request and response bodies the host does not control, + * so an over-limit frame is an expected condition, not a programming error. Every + * write path that can carry a large body uses this function, so no path emits a + * frame the peer decoder rejects. The bound applies to every frame type; the + * guard is a no-op for a small control frame. + */ +export function encodeDuplexFrameChecked( + frame: DuplexFrame, + maxFrameBytes: number = DEFAULT_MAX_DUPLEX_FRAME_BYTES, +): DuplexEncodeResult { + const json = JSON.stringify(frame); + if (Buffer.byteLength(json, "utf8") > maxFrameBytes) { + return { ok: false, error: { code: "frame_too_large", message: "frame exceeds the maximum size" } }; + } + return { ok: true, line: `${json}\n` }; +} + /** * Decode one line (no trailing newline) to a frame or a protocol error. The * streaming decoder calls this for each complete line. It is exported so a diff --git a/packages/adapter-utils/src/duplex-frame-vectors.json b/packages/adapter-utils/src/duplex-frame-vectors.json index 74ded507fad..5bedc93081e 100644 --- a/packages/adapter-utils/src/duplex-frame-vectors.json +++ b/packages/adapter-utils/src/duplex-frame-vectors.json @@ -414,5 +414,55 @@ } ] } + ], + "encodeDescription": "Encode-bound vectors. Each codec copy runs the size-checked encode against the frame with the maxFrameBytes limit, then asserts the same result. An ok result must decode back to the same frame. This proves both copies enforce the same bound on encode.", + "encodeVectors": [ + { + "name": "encode-within-bound", + "maxFrameBytes": 200, + "frame": { + "version": 1, + "type": "request", + "id": "req-enc-1", + "method": "GET", + "path": "/", + "query": "", + "headers": {}, + "body": "small" + }, + "expected": { + "ok": true + } + }, + { + "name": "encode-control-frame-never-rejected", + "maxFrameBytes": 200, + "frame": { + "version": 1, + "type": "heartbeat" + }, + "expected": { + "ok": true + } + }, + { + "name": "encode-over-bound", + "maxFrameBytes": 200, + "frame": { + "version": 1, + "type": "response", + "id": "req-enc-2", + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "outcome": "completed" + }, + "expected": { + "ok": false, + "error": "frame_too_large" + } + } ] } diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 0fd7cb426ed..73660142f9f 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -4842,8 +4842,13 @@ interface EmbeddedDecodeResult { // The names the embedded codec source declares. A test wraps the source and // reads these names back. +type EmbeddedEncodeResult = + | { ok: true; line: string } + | { ok: false; error: { code: string; message: string } }; + interface EmbeddedCodec { encodeDuplexFrame: (frame: unknown) => string; + encodeDuplexFrameChecked: (frame: unknown, maxFrameBytes?: number) => EmbeddedEncodeResult; decodeDuplexLine: (line: string | Buffer) => EmbeddedDecodeResult; DuplexFrameDecoder: new (options?: { maxFrameBytes?: number }) => { push: (chunk: Buffer) => EmbeddedDecodeResult[]; @@ -4864,10 +4869,18 @@ interface DuplexFrameVector { expected: ExpectedVectorResult[]; } +interface DuplexEncodeVector { + name: string; + maxFrameBytes: number; + frame: unknown; + expected: { ok: true } | { ok: false; error: string }; +} + interface DuplexFrameFixture { frameVersion: number; defaultMaxFrameBytes: number; vectors: DuplexFrameVector[]; + encodeVectors: DuplexEncodeVector[]; } describe("sandbox duplex gateway", () => { @@ -5086,7 +5099,7 @@ describe("sandbox duplex gateway", () => { it("embedded gateway codec passes every vector in the shared fixture", async () => { const codecFactory = new Function( - `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES };`, + `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES };`, ) as unknown as () => EmbeddedCodec; const codec = codecFactory(); @@ -5152,6 +5165,38 @@ describe("sandbox duplex gateway", () => { expect(decoded.ok).toBe(true); expect(decoded.frame).toEqual(want.frame); } + + // The embedded copy enforces the same encode bound as the host copy. It runs + // every shared encode vector and matches the expected result, so both copies + // reject the same oversized frame with the same `frame_too_large` code. + expect(fixture.encodeVectors.length).toBeGreaterThanOrEqual(3); + const encodeFailures: string[] = []; + for (const vector of fixture.encodeVectors) { + const result = codec.encodeDuplexFrameChecked(vector.frame, vector.maxFrameBytes); + if (result.ok !== vector.expected.ok) { + encodeFailures.push(`${vector.name}: ok=${result.ok}, want ${vector.expected.ok}`); + continue; + } + if (result.ok) { + if (!result.line.endsWith("\n") || result.line.slice(0, -1).includes("\n")) { + encodeFailures.push(`${vector.name}: line is not exactly one frame line`); + continue; + } + const decoded = codec.decodeDuplexLine(result.line.slice(0, -1)); + if (!decoded.ok) { + encodeFailures.push(`${vector.name}: an ok encode did not decode back`); + continue; + } + try { + expect(decoded.frame).toEqual(vector.frame); + } catch { + encodeFailures.push(`${vector.name}: decoded frame does not match`); + } + } else if (!vector.expected.ok && result.error.code !== vector.expected.error) { + encodeFailures.push(`${vector.name}: error ${result.error.code} != ${vector.expected.error}`); + } + } + expect(encodeFailures).toEqual([]); }); it("returns the same HTTP response as the file gateway for a forwarded request", async () => { @@ -5216,6 +5261,55 @@ describe("sandbox duplex gateway", () => { await gateway.stop(); }, 20000); + it("fails a request that exceeds the frame size bound with a local 413 and keeps the channel open", async () => { + const token = "duplex-token-oversize-request"; + // Raise the body limit above the frame bound, so `readBody` accepts the body + // and the encode guard is the only limit the request meets. The default frame + // bound is 1,000,000 bytes. + const gateway = await startDuplexGateway({ + PAPERCLIP_BRIDGE_TOKEN: token, + PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "3000000", + }); + + // A body over the frame bound makes the request frame exceed the bound. The + // gateway must fail this one local request with a clean 413 and forward no + // frame. + const oversizeBody = JSON.stringify({ body: "x".repeat(1_100_000) }); + const tooLarge = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: oversizeBody, + }); + expect(tooLarge.status).toBe(413); + await expect(tooLarge.json()).resolves.toEqual({ error: "request_too_large" }); + + // The oversized request forwarded no frame. It never left the gateway. + expect(gateway.frames.filter((frame) => frame.type === "request")).toHaveLength(0); + // The gateway did not lose the channel: no close frame and the process runs. + expect(gateway.frames.some((frame) => frame.type === "close")).toBe(false); + + // The channel stays open, so a normal request after the oversized one still + // forwards and completes. + const okPromise = fetch(`${gateway.baseUrl}/api/agents/me`, { + headers: { authorization: `Bearer ${token}` }, + }); + const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); + gateway.sendFrame({ + version: 1, + type: "response", + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ok: true }), + outcome: "completed", + }); + const okResponse = await okPromise; + expect(okResponse.status).toBe(200); + await expect(okResponse.json()).resolves.toEqual({ ok: true }); + + await gateway.stop(); + }, 20000); + it("enforces the bearer check, JSON-only rule, body limit, and depth limit", async () => { const token = "duplex-token-contract"; const gateway = await startDuplexGateway({ diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index fc5601708aa..7c81cfde263 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -1774,6 +1774,15 @@ function encodeDuplexFrame(frame) { return JSON.stringify(frame) + "\\n"; } +function encodeDuplexFrameChecked(frame, maxFrameBytes) { + const limit = maxFrameBytes != null ? maxFrameBytes : DEFAULT_MAX_DUPLEX_FRAME_BYTES; + const json = JSON.stringify(frame); + if (Buffer.byteLength(json, "utf8") > limit) { + return { ok: false, error: { code: "frame_too_large", message: "frame exceeds the maximum size" } }; + } + return { ok: true, line: json + "\\n" }; +} + function decodeDuplexLine(line) { const text = typeof line === "string" ? line : line.toString("utf8"); let parsed; @@ -1915,9 +1924,10 @@ class DuplexFrameDecoder { * Return the exact zero-dependency codec source the generated duplex gateway * embeds. A test runs every fixture vector against this source, so it proves the * embedded copy decodes the same bytes as the host codec. The source declares - * `encodeDuplexFrame`, `decodeDuplexLine`, `DuplexFrameDecoder`, - * `DUPLEX_FRAME_VERSION`, and `DEFAULT_MAX_DUPLEX_FRAME_BYTES`, but exports none - * of them; a caller wraps it to read those names. + * `encodeDuplexFrame`, `encodeDuplexFrameChecked`, `decodeDuplexLine`, + * `DuplexFrameDecoder`, `DUPLEX_FRAME_VERSION`, and + * `DEFAULT_MAX_DUPLEX_FRAME_BYTES`, but exports none of them; a caller wraps it to + * read those names. */ export function getSandboxDuplexGatewayCodecSource(): string { return DUPLEX_GATEWAY_CODEC_SOURCE; @@ -2273,6 +2283,16 @@ function runDuplexGateway() { headers: normalizeHeaders(req.headers), body: requestBody, }; + // Enforce the frame size bound on encode. When the request body makes the + // frame exceed the bound, fail this one local request with a clean 413. Do + // not write the frame and do not trigger loss. The frame never leaves the + // gateway, so no other in-flight request is affected and the channel stays + // open. + const encodedRequest = encodeDuplexFrameChecked(requestFrame); + if (!encodedRequest.ok) { + writeJsonResponse(res, 413, { error: "request_too_large" }); + return; + } const response = await new Promise((resolve) => { const timer = setTimeout(() => { if (pending.delete(requestId)) { @@ -2284,7 +2304,7 @@ function runDuplexGateway() { } }, responseTimeoutMs); pending.set(requestId, { resolve: resolve, timer: timer }); - writeFrame(requestFrame); + process.stdout.write(encodedRequest.line); }); res.statusCode = typeof response.status === "number" ? response.status : 200; for (const [key, value] of Object.entries(response.headers || {})) { From c5050396c7998fd29dbb6fcf16b2b3e9ae2771ea Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Sat, 22 Aug 2026 16:35:47 -0700 Subject: [PATCH 2/6] fix(daytona-duplex): chunk host-to-sandbox writes and make a transport close legible (#11986) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Paperclip runs agent work through adapters and sandbox providers > - The Daytona duplex path sends host input through a provider pseudo-terminal WebSocket > - Large messages exceed the provider limit, and a transport close can look like a process exit > - This pull request chunks UTF-8 input and carries transport-close state through the duplex path > - The benefit is reliable large input and accurate loss reporting ## Linked Issues or Issue Description **What happened?** The Daytona duplex path sent a full input payload as one WebSocket message. A payload above the provider limit closed the channel. The wait path also mapped a non-numeric exit result to a process exit without exit data. **Expected behavior** The provider must receive large input as ordered UTF-8 chunks. A transport close without exit data must record `transport_closed`, while a numeric exit must record `provider_exit`. **Steps to reproduce** 1. Start a Daytona duplex session. 2. Send an input payload larger than 65536 bytes. 3. Observe that one message closes the provider channel. 4. End a session without a numeric exit code. 5. Observe that the loss reason reports a process exit. **Paperclip version or commit** Commit `1761e79ec9097c65d94f90a8ba20416f8ab718a6`. **Deployment mode** Built from source with the Daytona sandbox provider. ## What Changed - Add a shared UTF-8 byte chunker with a 32768-byte cap. - Route both Daytona pseudo-terminal write paths through the chunker. - Preserve multi-byte UTF-8 sequences across read-side chunks. - Carry an explicit `transportClosed` state through the worker and host wait paths. - Record `transport_closed` for a reason-less transport close and `provider_exit` for a numeric exit. - Keep orderly completion suppression for both exit paths. ## Verification - The Daytona plugin suite passes 194 tests. - The adapter-utils broker, codec, and telemetry suites pass 73 tests. - The plugin SDK duplex and worker RPC host suites pass 37 tests. - The server plugin worker manager duplex suite passes 78 tests. - The execution target sandbox and ACPX execute suites pass 257 tests. - TypeScript checks pass for adapter-utils, plugin SDK, server, and the standalone Daytona plugin. ## Risks The chunk size adds a loop for large input payloads. The 32768-byte cap stays below the provider limit. The optional loss field preserves compatibility for other providers. ## Model Used OpenAI Codex, GPT-5, extended reasoning, tool use, and code execution. The runtime does not expose a separate context-window value. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs: #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/command-managed-runtime.ts | 9 +- .../src/duplex-bridge-broker.test.ts | 89 +++++++++++ .../adapter-utils/src/duplex-bridge-broker.ts | 23 ++- .../adapter-utils/src/duplex-telemetry.ts | 8 +- .../daytona/src/duplex-command-stream.test.ts | 142 +++++++++++++++++- .../daytona/src/duplex-command-stream.ts | 62 ++++++-- .../sandbox-providers/daytona/src/plugin.ts | 14 +- .../daytona/src/pty-chunked-input.test.ts | 138 +++++++++++++++++ .../daytona/src/pty-chunked-input.ts | 80 ++++++++++ .../daytona/src/setup-token-pty.ts | 17 ++- packages/plugins/sdk/src/protocol.ts | 6 + packages/plugins/sdk/src/types.ts | 8 +- packages/plugins/sdk/src/worker-rpc-host.ts | 11 +- packages/shared/src/telemetry/README.md | 1 + .../fixtures/plugin-worker-duplex-channel.cjs | 14 +- .../plugin-worker-manager-duplex.test.ts | 24 +++ server/src/services/environment-runtime.ts | 5 +- server/src/services/plugin-worker-manager.ts | 34 +++-- 18 files changed, 641 insertions(+), 44 deletions(-) create mode 100644 packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.test.ts create mode 100644 packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.ts diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index 4c879dbf4da..1121d8365d5 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -40,8 +40,13 @@ export interface CommandManagedDuplexChannel { write(data: string): void; /** Registers the one data listener. The channel streams each raw chunk in order. */ onData(listener: (chunk: string) => void): void; - /** Registers the one exit listener. The channel calls it one time with the exit. */ - onExit(listener: (exit: { exitCode: number | null }) => void): void; + /** + * Registers the one exit listener. The channel calls it one time with the exit. + * A numeric `exitCode` is a real process exit. `transportClosed` is true when the + * provider transport closed with no exit data, so a reader can tell a real + * process exit from a reason-less transport close. + */ + onExit(listener: (exit: { exitCode: number | null; transportClosed?: boolean }) => void): void; /** Stops the child process. Safe to call more than one time. */ stop(): void; /** Closes the channel and releases the route. Safe to call more than one time. */ diff --git a/packages/adapter-utils/src/duplex-bridge-broker.test.ts b/packages/adapter-utils/src/duplex-bridge-broker.test.ts index fccddb3b820..6c87a1ae877 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.test.ts @@ -16,6 +16,7 @@ import { } from "./duplex-frame-codec.js"; import { createDuplexTelemetry, + DUPLEX_COUNTER_LOSS_TOTAL, DUPLEX_SPAN_REQUEST, type DuplexTelemetryCounterRecord, type DuplexTelemetryEventRecord, @@ -855,3 +856,91 @@ describe("duplex bridge broker request limits", () => { }); }); }); + + +/** + * A channel harness that captures the broker exit listener. `emitExit` invokes it, + * so a test drives one channel exit into the broker. + */ +function createExitChannelHarness(): { + channel: CommandManagedDuplexChannel; + emitExit: (exit: { exitCode: number | null; transportClosed?: boolean }) => void; +} { + let exitListener: + | ((exit: { exitCode: number | null; transportClosed?: boolean }) => void) + | null = null; + const channel: CommandManagedDuplexChannel = { + write: () => undefined, + onData: () => undefined, + onExit: (listener) => { + exitListener = listener; + }, + stop: () => undefined, + close: () => Promise.resolve(), + }; + return { + channel, + emitExit: (exit) => { + if (!exitListener) throw new Error("The broker did not bind the exit listener."); + exitListener(exit); + }, + }; +} + +describe("duplex bridge broker exit taxonomy", () => { + const brokers: DuplexBridgeBroker[] = []; + + afterEach(async () => { + while (brokers.length > 0) { + const broker = brokers.pop(); + if (broker) await broker.close(); + } + }); + + it("records a transport close as a distinct loss, not a process exit", () => { + const { telemetry, capture } = createTelemetryCapture(); + const harness = createExitChannelHarness(); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: () => + Promise.resolve({ status: 200, headers: {}, body: "" }), + telemetry, + }); + brokers.push(broker); + broker.start(); + + // A reason-less transport close carries the discriminator and no exit code. + harness.emitExit({ exitCode: null, transportClosed: true }); + + expect(broker.lossRecord?.reason).toBe("transport_closed"); + expect(broker.runDisposition).toEqual({ failed: true, lossReason: "transport_closed" }); + const lossCounter = capture.counters.find( + (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, + ); + expect(lossCounter?.dimensions.loss_reason).toBe("transport_closed"); + }); + + it("records a numeric exit as a process exit", () => { + const { telemetry, capture } = createTelemetryCapture(); + const harness = createExitChannelHarness(); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: () => + Promise.resolve({ status: 200, headers: {}, body: "" }), + telemetry, + }); + brokers.push(broker); + broker.start(); + + // A numeric exit code is a real process exit, so the broker keeps the existing + // channel_exit -> provider_exit mapping. + harness.emitExit({ exitCode: 0 }); + + expect(broker.lossRecord?.reason).toBe("channel_exit"); + expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); + const lossCounter = capture.counters.find( + (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, + ); + expect(lossCounter?.dimensions.loss_reason).toBe("provider_exit"); + }); +}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts index 43d7b524f37..1a902360a9b 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.ts @@ -66,6 +66,7 @@ export type DuplexBrokerState = "opening" | "open" | "lost" | "closing" | "close /** The reason the broker classified a loss. The broker records it for metrics only. */ export type DuplexBrokerLossReason = | "channel_exit" + | "transport_closed" | "stream_failure" | "protocol_failure" | "heartbeat_write_failure" @@ -77,6 +78,8 @@ export type DuplexBrokerLossReason = * carries only the closed enum value. The map is total over the internal reasons, * so no raw text ever reaches the typed reason. * - `channel_exit` -> `provider_exit`: the provider channel process exited. + * - `transport_closed` -> `transport_closed`: the provider transport closed with + * no exit data, so the loss is a transport close, not a process exit. * - `stream_failure` -> `write_error`: a write to the channel failed. * - `protocol_failure` -> `rpc_failure`: a malformed or mismatched frame. * - `heartbeat_write_failure` -> `heartbeat_timeout`: the liveness write failed. @@ -84,6 +87,7 @@ export type DuplexBrokerLossReason = */ const BROKER_LOSS_REASON_TO_TYPED: Readonly> = { channel_exit: "provider_exit", + transport_closed: "transport_closed", stream_failure: "write_error", protocol_failure: "rpc_failure", heartbeat_write_failure: "heartbeat_timeout", @@ -457,10 +461,12 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr if (stopped) return; const afterOrderlyCompletion = orderlyCompletionSeq !== null; // A clean channel end that orders after a host-observed orderly completion is - // a normal teardown, not a loss. Stop cleanly, emit no loss event, and leave - // the run a success. This keeps the closed telemetry contract: an orderly - // close is not a loss and emits no loss event. - if (afterOrderlyCompletion && reason === "channel_exit") { + // a normal teardown, not a loss. A process exit and a reason-less transport + // close both end the channel, so both count as the normal teardown here. Stop + // cleanly, emit no loss event, and leave the run a success. This keeps the + // closed telemetry contract: an orderly close is not a loss and emits no loss + // event. + if (afterOrderlyCompletion && (reason === "channel_exit" || reason === "transport_closed")) { stopped = true; clearHeartbeat(); clearPending(); @@ -909,7 +915,14 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } }; - const onExit = (): void => { + const onExit = (exit: { exitCode: number | null; transportClosed?: boolean }): void => { + // A reason-less transport close is not a process exit. Record it as a distinct + // loss, so a transport close stays legible in the loss taxonomy. A real process + // exit stays `channel_exit` -> `provider_exit`. + if (exit.transportClosed === true) { + recordLoss("transport_closed", "The sandbox channel transport closed with no exit."); + return; + } recordLoss("channel_exit", "The sandbox channel process exited."); }; diff --git a/packages/adapter-utils/src/duplex-telemetry.ts b/packages/adapter-utils/src/duplex-telemetry.ts index 4bc08abebf6..2bcf8073cbd 100644 --- a/packages/adapter-utils/src/duplex-telemetry.ts +++ b/packages/adapter-utils/src/duplex-telemetry.ts @@ -85,9 +85,10 @@ export type DuplexLossClass = "pre_dispatch" | "post_dispatch"; * cause to one of these values before any sink reads it. The set covers the * loss-detection modes the transport names: a gateway stdin end of file, a * provider process exit, a heartbeat timeout, and an RPC failure. It adds - * `write_error` for a rejected host-to-sandbox write and `other` for an unknown - * cause. The host maps an unknown cause or any caught provider text to `other`, - * so no raw provider text reaches a sink. + * `write_error` for a rejected host-to-sandbox write, `transport_closed` for a + * reason-less provider transport close with no exit data, and `other` for an + * unknown cause. The host maps an unknown cause or any caught provider text to + * `other`, so no raw provider text reaches a sink. */ export const DUPLEX_LOSS_REASONS = [ "stdin_eof", @@ -95,6 +96,7 @@ export const DUPLEX_LOSS_REASONS = [ "heartbeat_timeout", "rpc_failure", "write_error", + "transport_closed", "other", ] as const; diff --git a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts index 1610a3d83e2..19339783f52 100644 --- a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts @@ -7,6 +7,7 @@ import { type DaytonaPtyHandle, type DaytonaPtyProcess, } from "./duplex-command-stream.js"; +import { PTY_INPUT_CHUNK_BYTES, PTY_MESSAGE_CAP_BYTES } from "./pty-chunked-input.js"; // The carriage return that submits the launch wrapper line to the terminal. const ENTER = "\r"; @@ -273,6 +274,8 @@ describe("openDaytonaDuplexChannelSession", () => { // The launch wrapper is the first input; the host frame write is the second. session.write('{"version":1,"type":"heartbeat"}\n'); + // The chunker awaits each send, so let the write settle before the assertion. + await new Promise((resolve) => setImmediate(resolve)); expect(process.handle?.inputs[1]).toBe('{"version":1,"type":"heartbeat"}\n'); // The fake terminal echoes nothing, so the write produces no data callback. // A real terminal in raw mode with echo off behaves the same way. @@ -357,22 +360,104 @@ describe("openDaytonaDuplexChannelSession", () => { expect(received).toEqual(['{"version":1,"type":"heartbeat"}\n']); }); - it("resolves wait with the gateway exit code", async () => { + it("maps a numeric SDK exit code to a process exit", async () => { const process = createFakeProcess(); const session = await openDaytonaDuplexChannelSession(process, GATEWAY); process.handle?.finish(3); - await expect(session.wait()).resolves.toEqual({ exitCode: 3 }); + // A numeric exit code is a real process exit, not a transport close. + await expect(session.wait()).resolves.toEqual({ exitCode: 3, transportClosed: false }); }); - it("maps an absent exit code to null", async () => { + it("maps an absent SDK exit code to a transport close", async () => { const process = createFakeProcess(); const session = await openDaytonaDuplexChannelSession(process, GATEWAY); process.handle?.finish(undefined); - await expect(session.wait()).resolves.toEqual({ exitCode: null }); + // A non-numeric SDK result marks a reason-less transport close with no exit + // data, so the seam reports a transport close, not a process exit. + await expect(session.wait()).resolves.toEqual({ exitCode: null, transportClosed: true }); + }); + + it("splits a write above the provider cap into more than one send under the cap", async () => { + const process = createFakeProcess(); + const session = await openDaytonaDuplexChannelSession(process, GATEWAY); + const handle = process.handle; + if (!handle) throw new Error("The fake process opened no handle."); + // The launch wrapper is the first recorded input. Count only the write's sends. + const before = handle.inputs.length; + + // A fixed payload well above the provider message cap. It stays fixed, so a + // raised chunk size above this size makes the write one send and fails this + // test. The payload is ASCII, so the fake's per-send decode keeps each byte. + const payload = "x".repeat(150_416); + session.write(payload); + // The chunker awaits each send, so let the write settle before the assertion. + await new Promise((resolve) => setImmediate(resolve)); + + const sends = handle.inputs.slice(before); + expect(sends.length).toBeGreaterThan(1); + for (const chunk of sends) { + expect(Buffer.byteLength(chunk, "utf8")).toBeLessThanOrEqual(PTY_INPUT_CHUNK_BYTES); + expect(Buffer.byteLength(chunk, "utf8")).toBeLessThanOrEqual(PTY_MESSAGE_CAP_BYTES); + } + // The sends rejoin to the exact payload, so the chunker loses no byte. + expect(sends.join("")).toBe(payload); + }); + + it("sends a write below the provider cap as exactly one send", async () => { + const process = createFakeProcess(); + const session = await openDaytonaDuplexChannelSession(process, GATEWAY); + const handle = process.handle; + if (!handle) throw new Error("The fake process opened no handle."); + const before = handle.inputs.length; + + session.write('{"version":1,"type":"heartbeat"}\n'); + // The chunker awaits each send, so let the write settle before the assertion. + await new Promise((resolve) => setImmediate(resolve)); + + expect(handle.inputs.slice(before)).toHaveLength(1); + }); + + it("keeps two writes in order and never interleaves their chunks", async () => { + // A handle whose `sendInput` yields a microtask before it records the chunk. + // The yield opens a window where a second write could jump ahead. The session + // chains the writes, so the second write starts only after the first ends. + const inputs: string[] = []; + const handle: DaytonaPtyHandle = { + async waitForConnection(): Promise {}, + async sendInput(data: string | Uint8Array): Promise { + await Promise.resolve(); + inputs.push(typeof data === "string" ? data : new TextDecoder().decode(data)); + }, + wait(): Promise<{ exitCode?: number; error?: string }> { + return new Promise(() => {}); + }, + async kill(): Promise {}, + async disconnect(): Promise {}, + }; + const process: DaytonaPtyProcess = { + async createPty(): Promise { + return handle; + }, + }; + const session = await openDaytonaDuplexChannelSession(process, GATEWAY); + // Two payloads above the cap, so each write is more than one chunk. "a" marks + // the first write and "b" marks the second write. + const first = "a".repeat(150_416); + const second = "b".repeat(150_416); + + session.write(first); + session.write(second); + // Let both writes settle. + await new Promise((resolve) => setImmediate(resolve)); + + // Drop the launch wrapper (the first input) and rejoin the chunks. The rejoin + // equals the first payload then the second payload, so no chunk of the second + // write lands between the chunks of the first write. + expect(inputs.slice(1).join("")).toBe(`${first}${second}`); }); it("kills the child on stop and releases the socket on close", async () => { @@ -475,6 +560,55 @@ describe("openDaytonaDuplexChannelSession", () => { expect(writeErrors).toEqual(["write_error"]); expect(killed).toBe(1); }); + + it("does not send a queued write after a first-send rejection terminalizes the channel", async () => { + // Two writes queue on the chain. The launch wrapper write succeeds, then the + // first frame write rejects and terminalizes the channel. The second queued + // write must not reach `sendInput`, so no write runs on the closed transport. + const sends: string[] = []; + let killed = 0; + const handle: DaytonaPtyHandle = { + async waitForConnection(): Promise {}, + async sendInput(data: string | Uint8Array): Promise { + const text = typeof data === "string" ? data : new TextDecoder().decode(data); + sends.push(text); + // The launch wrapper write (the first send) succeeds. The first frame + // write (the second send) rejects and terminalizes the channel. + if (sends.length === 1) return; + throw new Error("broken"); + }, + wait(): Promise<{ exitCode?: number; error?: string }> { + return new Promise(() => {}); + }, + async kill(): Promise { + killed += 1; + }, + async disconnect(): Promise {}, + }; + const process: DaytonaPtyProcess = { + async createPty(): Promise { + return handle; + }, + }; + const writeErrors: string[] = []; + const session = await openDaytonaDuplexChannelSession(process, GATEWAY, { + onWriteError: (reason) => writeErrors.push(reason), + }); + + session.write("first\n"); + session.write("second\n"); + // Let the write chain settle both queued writes. + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + // The channel terminalized one time on the first frame write. The second + // queued write sent no chunk, so exactly two sends ran: the launch wrapper and + // the first frame write. The second frame never reached the transport. + expect(writeErrors).toEqual(["write_error"]); + expect(killed).toBe(1); + expect(sends).toHaveLength(2); + expect(sends.some((text) => text.includes("second"))).toBe(false); + }); }); describe("createDaytonaDuplexChannelSessionOpener", () => { diff --git a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts index 2aba7b76c8c..4507ba774e8 100644 --- a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts +++ b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts @@ -45,6 +45,8 @@ export type { DaytonaPtyProcess, } from "./setup-token-pty.js"; +import { sendPtyInputInChunks } from "./pty-chunked-input.js"; + /** * A live duplex channel session for one command. The session allocates a real * pseudo-terminal in raw mode, streams the raw output, accepts host input, and @@ -56,8 +58,13 @@ export interface DuplexChannelSession { onData(listener: (chunk: string) => void): void; /** Writes raw input bytes to the pseudo-terminal. */ write(data: string): void; - /** Resolves with the child exit code when the command ends. */ - wait(): Promise<{ exitCode: number | null }>; + /** + * Resolves when the command ends or the transport closes. A numeric `exitCode` + * is a real process exit. `transportClosed` is true when the pseudo-terminal + * socket closed with no exit data, so the caller can tell a real process exit + * from a reason-less transport close. + */ + wait(): Promise<{ exitCode: number | null; transportClosed: boolean }>; /** Stops the child process. Safe to call more than one time. */ kill(): void; /** Releases the session resources. Safe to call more than one time. */ @@ -203,14 +210,28 @@ export async function openDaytonaDuplexChannelSession( // Report a host-to-sandbox write failure one time and end the channel at once. // The seam carries only the typed reason; the raw provider error never leaves // this scope. The channel end propagates the loss up through the exit. + // + // `channelTerminated` turns true on the first rejected chunk, before the early + // return, so a later queued write reads it and sends no chunk. Without the flag + // a queued write runs after the terminal kill and calls `sendInput` on the + // closed transport. + let channelTerminated = false; let writeErrorReported = false; const endOnWriteError = (): void => { + channelTerminated = true; if (writeErrorReported) return; writeErrorReported = true; options?.onWriteError?.(DAYTONA_DUPLEX_WRITE_ERROR_REASON); void handle.kill().catch(() => undefined); }; + // The tail of the write chain. The chunker awaits each chunk, so one write can + // suspend between its chunks. The chain runs each write after the previous + // write ends, so the chunks of two writes never interleave on the wire. The + // chain never rejects, because the chunker maps a rejected send to + // `endOnWriteError` and returns. + let writeChain: Promise = Promise.resolve(); + return { onData(next: (chunk: string) => void): void { listener = next; @@ -221,17 +242,38 @@ export async function openDaytonaDuplexChannelSession( } }, write(data: string): void { - // Fire the input write. A write error must not throw into the transport, so - // the transport's stream stays the single result path. On a rejected write, - // end the channel at once and report the typed reason. The raw provider - // error never reaches a sink. - void handle.sendInput(data).catch(() => { - endOnWriteError(); + // Send the input as byte-bounded chunks under the provider message cap. A + // whole payload in one message can cross the cap and take the channel down, + // so the chunker slices the payload and sends each chunk in order. The chain + // runs this write after the previous write ends, so two writes keep their + // order and never interleave their chunks. A write error must not throw into + // the transport, so the transport's stream stays the single result path. On + // a rejected chunk, the chunker ends the channel one time through + // `endOnWriteError` and sends no later chunk. The raw provider error never + // reaches a sink. + // + // A rejected chunk terminalizes the channel and kills the transport. So this + // write skips the send when `channelTerminated` is true, and a queued write + // never calls `sendInput` on the closed transport. + writeChain = writeChain.then(() => { + if (channelTerminated) return; + return sendPtyInputInChunks( + (chunk) => handle.sendInput(chunk), + data, + endOnWriteError, + ); }); }, - async wait(): Promise<{ exitCode: number | null }> { + async wait(): Promise<{ exitCode: number | null; transportClosed: boolean }> { const result = await handle.wait(); - return { exitCode: typeof result.exitCode === "number" ? result.exitCode : null }; + if (typeof result.exitCode === "number") { + // A numeric exit code is a real process exit. The SDK parses it from the + // pseudo-terminal WebSocket close reason (for example `{"exitCode":0}`). + return { exitCode: result.exitCode, transportClosed: false }; + } + // A non-numeric exit code marks a reason-less transport close: the socket + // closed with no exit data. It is a transport close, not a process exit. + return { exitCode: null, transportClosed: true }; }, kill(): void { void handle.kill().catch(() => undefined); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 304947e562f..88f9f060328 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -2828,9 +2828,19 @@ const plugin = definePlugin({ session.onData((chunk) => { pluginContext?.duplexChannel.data(entry.hostRouteId, workerSessionId, chunk); }); - // Forward the child exit one time. The host resolves the open route on it. + // Forward the child exit one time. The host resolves the open route on it. A + // numeric exit code is a real process exit; a resolved transport close carries + // `transportClosed`, so the host keeps the two apart in the loss taxonomy. A + // rejected wait is not a resolved transport close, so it reports no code and no + // transport-close mark. void session.wait().then( - (result) => pluginContext?.duplexChannel.exit(entry.hostRouteId, workerSessionId, result.exitCode), + (result) => + pluginContext?.duplexChannel.exit( + entry.hostRouteId, + workerSessionId, + result.exitCode, + result.transportClosed, + ), () => pluginContext?.duplexChannel.exit(entry.hostRouteId, workerSessionId, null), ); // Echo the host route id on the reply, so the host binds the exact pair. diff --git a/packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.test.ts b/packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.test.ts new file mode 100644 index 00000000000..cc1fbbbce55 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; +import { + PTY_INPUT_CHUNK_BYTES, + PTY_MESSAGE_CAP_BYTES, + sendPtyInputInChunks, +} from "./pty-chunked-input.js"; + +/** + * A fake `sendInput`. It records each raw byte chunk, so a test asserts the chunk + * count, the chunk sizes, and the rejoined bytes. It copies each chunk, because + * the chunker passes a `subarray` view of one shared byte array. + */ +function createByteRecorder(): { + sendInput: (chunk: Uint8Array) => Promise; + chunks: Uint8Array[]; +} { + const chunks: Uint8Array[] = []; + return { + chunks, + async sendInput(chunk: Uint8Array): Promise { + chunks.push(Uint8Array.from(chunk)); + }, + }; +} + +/** Concatenate the recorded chunks into one byte array. */ +function concatChunks(chunks: readonly Uint8Array[]): Uint8Array { + const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const joined = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + joined.set(chunk, offset); + offset += chunk.length; + } + return joined; +} + +describe("sendPtyInputInChunks", () => { + it("sends a payload below the cap as exactly one send", async () => { + const recorder = createByteRecorder(); + + await sendPtyInputInChunks(recorder.sendInput, "a small frame\n"); + + expect(recorder.chunks).toHaveLength(1); + expect(recorder.chunks[0]?.length).toBeLessThanOrEqual(PTY_MESSAGE_CAP_BYTES); + }); + + it("splits a payload above the cap into more than one send, each at or below the cap", async () => { + const recorder = createByteRecorder(); + // A fixed payload well above the provider message cap. It stays fixed, so a + // raised chunk size above this size makes the payload one send and fails this + // test. + const payload = new Uint8Array(150_416).fill(65); + + await sendPtyInputInChunks(recorder.sendInput, payload); + + expect(recorder.chunks.length).toBeGreaterThan(1); + for (const chunk of recorder.chunks) { + expect(chunk.length).toBeLessThanOrEqual(PTY_INPUT_CHUNK_BYTES); + expect(chunk.length).toBeLessThanOrEqual(PTY_MESSAGE_CAP_BYTES); + } + expect(concatChunks(recorder.chunks)).toEqual(payload); + }); + + it("slices a multi-byte character across a boundary by bytes and rejoins it", async () => { + const recorder = createByteRecorder(); + // A fixed payload: one 1-byte "x", then the 3-byte character "€" repeated. It + // stays fixed, so a raised chunk size above this size makes the payload one + // send and fails this test. The chunk size is a multiple of 3, so a pure "€" + // run would put every boundary on a character edge. The 1-byte prefix shifts + // the "€" run by one byte, so the first chunk boundary lands inside one "€" + // sequence. + const text = `x${"€".repeat(30_000)}`; + const inputBytes = new TextEncoder().encode(text); + expect(inputBytes.length).toBe(90_001); + + await sendPtyInputInChunks(recorder.sendInput, text); + + expect(recorder.chunks.length).toBeGreaterThan(1); + // The first chunk holds exactly the byte cap and ends inside a "€" sequence, + // because the 1-byte prefix shifts the 3-byte run off the cap boundary. A + // character-index slice could never end mid-sequence, so the slice is by bytes. + expect(recorder.chunks[0]?.length).toBe(PTY_INPUT_CHUNK_BYTES); + // The rejoined bytes equal the input bytes, so a split multi-byte sequence is + // whole again. A streaming decoder on the read side then decodes it. + const joined = concatChunks(recorder.chunks); + expect(joined).toEqual(inputBytes); + expect(new TextDecoder("utf-8").decode(joined)).toBe(text); + }); + + it("resolves each send before it starts the next send", async () => { + let active = 0; + let maxActive = 0; + const sendCount = { value: 0 }; + // A `sendInput` that stays active across a microtask. The awaited loop starts + // the next send only after the previous send resolves, so at most one send is + // active at a time. A synchronous burst would start every send at once. + const sendInput = async (): Promise => { + active += 1; + maxActive = Math.max(maxActive, active); + await Promise.resolve(); + sendCount.value += 1; + active -= 1; + }; + // Three chunks of the byte cap plus a short tail. + const payload = new Uint8Array(PTY_INPUT_CHUNK_BYTES * 3 + 10).fill(65); + + await sendPtyInputInChunks(sendInput, payload); + + expect(sendCount.value).toBe(4); + expect(maxActive).toBe(1); + }); + + it("stops sending later chunks after a chunk rejects", async () => { + const chunks: Uint8Array[] = []; + let errorCount = 0; + // A `sendInput` that records the chunk, then rejects on a later microtask. The + // rejection is asynchronous, so a synchronous burst could not stop the later + // sends. The awaited loop learns the rejection before the next send, so it + // sends one chunk only. + const sendInput = async (chunk: Uint8Array): Promise => { + chunks.push(chunk); + await Promise.resolve(); + throw new Error("RAW-PROVIDER-BROKEN-PIPE"); + }; + + // A payload above the cap, so the chunker would fire more than one chunk with + // no fail-fast. The loop stops after the first rejected chunk. + await sendPtyInputInChunks(sendInput, new Uint8Array(150_416).fill(65), () => { + errorCount += 1; + }); + + // The loop sent one chunk, learned the rejection, and started no later send. + expect(chunks).toHaveLength(1); + // The error handler ran one time for the whole payload. + expect(errorCount).toBe(1); + }); +}); diff --git a/packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.ts b/packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.ts new file mode 100644 index 00000000000..dd9ee7e6272 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/pty-chunked-input.ts @@ -0,0 +1,80 @@ +// The shared host-to-sandbox write chunker for the Daytona pseudo-terminal (PTY) +// sessions. The provider PTY WebSocket rejects any single message over 65536 +// bytes with WebSocket close code 1009 and closes the connection. `sendInput` +// sends a whole payload in one message, so a payload above the cap takes the +// channel down. Real duplex frames cross the cap (single frames of 127582 and +// 150416 bytes are observed). This module slices each payload into byte-bounded +// chunks under the cap and sends each chunk in order. + +/** + * The provider PTY WebSocket message cap, in bytes. The provider rejects a single + * PTY message over this size with WebSocket close code 1009 and closes the + * connection. + */ +export const PTY_MESSAGE_CAP_BYTES = 65536; + +/** + * The maximum byte length of one host-to-sandbox chunk. It holds about 5 KiB of + * headroom under {@link PTY_MESSAGE_CAP_BYTES}, so a chunk never sits on the cap + * boundary. A later change in how the provider counts message overhead does not + * push one chunk over the cap and back into the close-code-1009 failure. The + * headroom costs about nine percent more messages for a large payload. + * + * The chunker slices bytes, not string characters, so a chunk boundary can split + * a multi-byte UTF-8 sequence. The read-side streaming `TextDecoder` rejoins a + * split sequence, so a byte slice is safe. A string-index slice could split a + * surrogate pair, so the chunker never slices by character. + */ +export const PTY_INPUT_CHUNK_BYTES = 60 * 1024; + +/** The single UTF-8 encoder the chunker reuses for every string payload. */ +const PTY_INPUT_TEXT_ENCODER = new TextEncoder(); + +/** + * Send one host-to-sandbox payload as byte-bounded chunks under the provider + * message cap. The function encodes a string payload to UTF-8 bytes, slices the + * byte array into chunks of at most {@link PTY_INPUT_CHUNK_BYTES}, and calls + * `sendInput` for each chunk in order. + * + * The loop awaits each chunk before it sends the next chunk. The `await` keeps + * the wire order equal to the slice order, and it makes the chunk send fail-fast: + * after a chunk rejects, the function sends no further chunk of this payload. A + * synchronous loop could not fail-fast, because `sendInput` reaches `ws.send` + * before its first `await`, so a synchronous loop puts every chunk on the wire + * before the first rejection settles. The awaited loop learns the rejection + * before it sends the next chunk, so it stops. + * + * The function does not overlap two calls for the same channel. The caller + * serializes its writes (for example, through a promise chain), so the awaited + * loop of one payload never interleaves its chunks with the chunks of the next + * payload. + * + * `onError` runs at most one time. The function reports the first rejected chunk + * and returns. A rejected send never throws out of this function, so the caller's + * stream stays the single result path. + * + * @param sendInput - Sends one raw byte chunk to the pseudo-terminal. + * @param payload - The host-to-sandbox payload, as a string or raw bytes. + * @param onError - Optional. The function calls it one time on the first rejected chunk. + */ +export async function sendPtyInputInChunks( + sendInput: (chunk: Uint8Array) => Promise, + payload: string | Uint8Array, + onError?: () => void, +): Promise { + const bytes = typeof payload === "string" ? PTY_INPUT_TEXT_ENCODER.encode(payload) : payload; + for (let offset = 0; offset < bytes.length; offset += PTY_INPUT_CHUNK_BYTES) { + const chunk = bytes.subarray(offset, offset + PTY_INPUT_CHUNK_BYTES); + try { + // Await the send before the next chunk. The await keeps the wire order and + // lets the loop learn a rejection before it starts a later send. + await sendInput(chunk); + } catch { + // Report the first rejected chunk one time and stop. The function starts no + // later send for this payload, so the wire keeps the ordered, fail-fast + // behavior. The raw provider error never leaves this function. + onError?.(); + return; + } + } +} diff --git a/packages/plugins/sandbox-providers/daytona/src/setup-token-pty.ts b/packages/plugins/sandbox-providers/daytona/src/setup-token-pty.ts index 89bace5b5ce..769e442a195 100644 --- a/packages/plugins/sandbox-providers/daytona/src/setup-token-pty.ts +++ b/packages/plugins/sandbox-providers/daytona/src/setup-token-pty.ts @@ -27,6 +27,8 @@ import { randomUUID } from "node:crypto"; +import { sendPtyInputInChunks } from "./pty-chunked-input.js"; + /** * A live pseudo-terminal session for one setup-token login command. The session * allocates a real pseudo-terminal, streams the raw terminal output, accepts @@ -151,6 +153,11 @@ export async function openDaytonaSetupTokenPtySession( // to the command, not to a shell. await handle.sendInput(`exec ${command}${PTY_COMMAND_TERMINATOR}`); + // The tail of the write chain. The chunker awaits each chunk, so one write can + // suspend between its chunks. The chain runs each write after the previous + // write ends, so the chunks of two writes never interleave on the wire. + let writeChain: Promise = Promise.resolve(); + return { onData(next: (chunk: string) => void): void { listener = next; @@ -161,9 +168,15 @@ export async function openDaytonaSetupTokenPtySession( } }, write(data: string): void { - // Fire the input write. A write error must not throw into the runner, so + // Send the input as byte-bounded chunks under the provider message cap. The + // login input is short today, so the latent defect never fires, but the same + // unbounded write goes through the one chunker. The chain runs this write + // after the previous write ends, so two writes keep their order and never + // interleave their chunks. A write error must not throw into the runner, so // the runner's fixed status stays the single result path. - void handle.sendInput(data).catch(() => undefined); + writeChain = writeChain.then(() => + sendPtyInputInChunks((chunk) => handle.sendInput(chunk), data), + ); }, async wait(): Promise<{ exitCode: number | null }> { const result = await handle.wait(); diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.ts index 4c98b410889..be1c5792564 100644 --- a/packages/plugins/sdk/src/protocol.ts +++ b/packages/plugins/sdk/src/protocol.ts @@ -1191,6 +1191,12 @@ export interface PluginDuplexChannelExitParams { workerSessionId: string; /** The child exit code, or null when the child ended with no code. */ exitCode: number | null; + /** + * True when the provider transport closed with no exit data, so the exit is a + * reason-less transport close, not a process exit. Absent or false marks a real + * process exit. The host maps a transport close to a distinct loss reason. + */ + transportClosed?: boolean; } /** The worker→host notification method for one duplex channel data chunk. */ diff --git a/packages/plugins/sdk/src/types.ts b/packages/plugins/sdk/src/types.ts index 8ea10b964eb..412054dc77d 100644 --- a/packages/plugins/sdk/src/types.ts +++ b/packages/plugins/sdk/src/types.ts @@ -2065,8 +2065,14 @@ export interface PluginDuplexChannelClient { * @param hostRouteId - The host route identifier the open request carried. The worker echoes it, so the host routes the exact pair. * @param workerSessionId - The worker session identifier the open reply returned. * @param exitCode - The child exit code, or null when the child ended with no code. + * @param transportClosed - True when the transport closed with no exit data, so the exit is a reason-less transport close, not a process exit. Absent marks a real process exit. */ - exit(hostRouteId: string, workerSessionId: string, exitCode: number | null): void; + exit( + hostRouteId: string, + workerSessionId: string, + exitCode: number | null, + transportClosed?: boolean, + ): void; } // --------------------------------------------------------------------------- diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index 901410a8411..82311ff504f 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -1415,16 +1415,23 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost if (typeof chunk !== "string" || chunk.length === 0) return; notifyHost(DUPLEX_CHANNEL_DATA_NOTIFICATION, { hostRouteId, workerSessionId, chunk }); }, - exit(hostRouteId: string, workerSessionId: string, exitCode: number | null): void { + exit( + hostRouteId: string, + workerSessionId: string, + exitCode: number | null, + transportClosed?: boolean, + ): void { // Forward the child exit of a persistent duplex channel. The host // resolves the open route's wait promise by the exact live pair while - // the route is open. + // the route is open. `transportClosed` carries the exit discriminator, so + // the host tells a real process exit from a reason-less transport close. if (typeof hostRouteId !== "string" || hostRouteId.length === 0) return; if (typeof workerSessionId !== "string" || workerSessionId.length === 0) return; notifyHost(DUPLEX_CHANNEL_EXIT_NOTIFICATION, { hostRouteId, workerSessionId, exitCode: typeof exitCode === "number" ? exitCode : null, + transportClosed: transportClosed === true, }); }, }, diff --git a/packages/shared/src/telemetry/README.md b/packages/shared/src/telemetry/README.md index 159501a8dc4..bba168e5cb7 100644 --- a/packages/shared/src/telemetry/README.md +++ b/packages/shared/src/telemetry/README.md @@ -366,6 +366,7 @@ key never reaches a sink by accident. | `outcome` | string | yes | `ok` or `error`. | | `fallback_reason` | string | yes | `gate_off`, `capability_absent`, `open_failed`, `ready_invalid`, `ready_nonce_mismatch`, `ready_timeout`, or `contaminated`. It rides only a fallback record. | | `loss_class` | string | yes | `pre_dispatch` or `post_dispatch`, relative to the first request dispatch. It rides only a loss record. | +| `loss_reason` | string | yes | `stdin_eof`, `provider_exit`, `heartbeat_timeout`, `rpc_failure`, `write_error`, `transport_closed`, or `other`. The host maps every loss cause to one of these values, so no raw provider text reaches a sink. `write_error` marks a rejected host-to-sandbox write. `transport_closed` marks a reason-less provider transport close with no exit data. It rides only a loss record. | To add a name or an enum value, extend the literal constant in `duplex-telemetry.ts` first, then update the test that asserts the closed set. diff --git a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs index 914df984dd8..013ecb03629 100644 --- a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs +++ b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs @@ -14,6 +14,8 @@ // `sid` or `rid` to prove the host drops a mismatched-pair notification and // counts a protocol error. // - `exitCode`: when set, the fixture emits an exit notification after the data. +// - `transportClosed`: when true, the exit notification marks a reason-less +// transport close and carries no exit code. // - `dataAfterExit`: an array of `{ chunk, sid? }`, same shape as `data`. The // fixture emits these as data notifications after the exit notification, so a // test can script a batch where a data frame arrives after the exit in the @@ -52,11 +54,19 @@ function scriptedFrameLines(directive, hostRouteId, workerSessionId) { }, })}\n`; } - if (typeof directive.exitCode === "number") { + if (typeof directive.exitCode === "number" || directive.transportClosed === true) { + const exitParams = { + hostRouteId, + workerSessionId, + exitCode: typeof directive.exitCode === "number" ? directive.exitCode : null, + }; + // A transport-close directive marks a reason-less transport close, so the + // fixture carries the discriminator the same way a real worker does. + if (directive.transportClosed === true) exitParams.transportClosed = true; lines += `${JSON.stringify({ jsonrpc: "2.0", method: "duplexChannel.exit", - params: { hostRouteId, workerSessionId, exitCode: directive.exitCode }, + params: exitParams, })}\n`; } const dataAfterExit = Array.isArray(directive.dataAfterExit) ? directive.dataAfterExit : []; diff --git a/server/src/__tests__/plugin-worker-manager-duplex.test.ts b/server/src/__tests__/plugin-worker-manager-duplex.test.ts index 85bc5f3dfff..0b746e8bd01 100644 --- a/server/src/__tests__/plugin-worker-manager-duplex.test.ts +++ b/server/src/__tests__/plugin-worker-manager-duplex.test.ts @@ -163,6 +163,30 @@ describe("plugin worker manager duplex channel route", () => { } }); + it("carries a transport-close exit through to the wait result", async () => { + const handle = makeDuplexHandle(); + try { + await handle.start(); + const session = await handle.openDuplexChannel( + duplexOpenInput({ + workerSessionId: "ws-A", + data: [{ chunk: "one" }], + // The worker reports a reason-less transport close with no exit code. + transportClosed: true, + }), + ); + const chunks: string[] = []; + session.onData((chunk) => chunks.push(chunk)); + // The discriminator survives the worker exit notification, so the host wait + // resolves with the transport-close mark and no exit code. + await expect(session.wait()).resolves.toEqual({ exitCode: null, transportClosed: true }); + expect(chunks).toEqual(["one"]); + await session.close(); + } finally { + await handle.stop().catch(() => undefined); + } + }); + it("isolates a throwing listener during the buffered replay so every buffered chunk routes", async () => { const handle = makeDuplexHandle(); try { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 6cf5cf33f21..9187a59aa83 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1080,9 +1080,10 @@ function adaptDuplexChannelHostSession( onData(listener: (chunk: string) => void): void { session.onData(listener); }, - onExit(listener: (exit: { exitCode: number | null }) => void): void { + onExit(listener: (exit: { exitCode: number | null; transportClosed?: boolean }) => void): void { // `wait()` resolves one time with the exit and never rejects, so a single - // `then` bridges it to the one-time exit listener. + // `then` bridges it to the one-time exit listener. The exit carries + // `transportClosed`, so the broker tells a real exit from a transport close. void session.wait().then((exit) => { listener(exit); }); diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index 76dea17f0c4..77ac1ff63a8 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -572,8 +572,13 @@ export interface DuplexChannelHostSession { onData(listener: (chunk: string) => void): void; /** Writes raw input bytes to the channel. */ write(data: string): void; - /** Resolves with the child exit code when the command ends or the route ends. */ - wait(): Promise<{ exitCode: number | null }>; + /** + * Resolves when the command ends or the route ends. A numeric `exitCode` is a + * real process exit. `transportClosed` is true when the provider transport + * closed with no exit data, so the broker can tell a real process exit from a + * reason-less transport close. + */ + wait(): Promise<{ exitCode: number | null; transportClosed?: boolean }>; /** Stops the child process. Safe to call more than one time. */ kill(): void; /** Closes the route and releases the channel. Safe to call more than one time. */ @@ -1280,8 +1285,8 @@ export function createPluginWorkerHandle( // Settle the route wait exactly once. Replace the settler with a no-op, so a // later exit or terminalize never settles the wait a second time. function settleRouteWait( - route: { settleWait: (value: { exitCode: number | null }) => void }, - value: { exitCode: number | null }, + route: { settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void }, + value: { exitCode: number | null; transportClosed?: boolean }, ): void { const settle = route.settleWait; route.settleWait = () => {}; @@ -1557,7 +1562,7 @@ export function createPluginWorkerHandle( totalDataBytes: number; lifetimeTimer: ReturnType | null; terminalized: boolean; - settleWait: (value: { exitCode: number | null }) => void; + settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void; // The data frames that arrived before the bind, held in order. The bind // replays them through the exact-pair routing, so an early frame is never // lost and a frame whose pair does not match the bound pair still fails @@ -1869,6 +1874,14 @@ export function createPluginWorkerHandle( return; } const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; + // A reason-less transport close carries `transportClosed`; a real process exit + // does not. Carry the discriminator to the wait only when it is a transport + // close, so a real exit keeps the plain `{ exitCode }` result and the broker + // still keeps the two apart in the loss taxonomy. + if (params.transportClosed === true) { + settleRouteWait(resolved, { exitCode, transportClosed: true }); + return; + } settleRouteWait(resolved, { exitCode }); } @@ -1953,10 +1966,13 @@ export function createPluginWorkerHandle( input: DuplexChannelOpenInput, ): Promise { const hostRouteId = nextDuplexHostRouteId(); - let settleWait: (value: { exitCode: number | null }) => void = () => {}; - const waitPromise = new Promise<{ exitCode: number | null }>((resolve) => { - settleWait = resolve; - }); + let settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void = + () => {}; + const waitPromise = new Promise<{ exitCode: number | null; transportClosed?: boolean }>( + (resolve) => { + settleWait = resolve; + }, + ); const route: DuplexChannelRoute = { hostRouteId, state: "reserved", From 05b35d4669cebea2e1d0bad194caf883b43d8550 Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Sat, 22 Aug 2026 22:14:35 -0700 Subject: [PATCH 3/6] feat(duplex): bound aggregate duplex route resource consumption with a process-owned byte ledger (#12003) ## Thinking Path > - Paperclip runs AI agents through adapters and sandboxed execution targets. > - Duplex routes retain bytes across route data, broker messages, decoder buffers, and readiness replay. > - Per-route limits bound each route but do not bound the total retained bytes across many routes. > - A process-owned ledger must charge each retained buffer before allocation and release the charge during cleanup. > - This pull request adds the aggregate ledger, connects it to host and sandbox duplex paths, and adds route coverage. > - The benefit is a fail-closed process-wide byte limit that keeps concurrent duplex work within a safe resource budget. ## Linked Issues or Issue Description **Subsystem affected** This change affects packages/adapter-utils and server duplex orchestration. **Problem or motivation** Many routes can each stay below their per-route limits while their combined retained bytes exceed a safe process budget. **Proposed solution** Add a process-owned aggregate byte ledger. Charge route data, broker bytes, decoder buffers, and readiness replay bytes before allocation. Release each charge during cleanup. Use a separate sandbox_process decoder cap for the in-sandbox path. **Alternatives considered** Keep only per-route limits. This does not bound the combined process use. Set a fixed limit at one call site. This misses retained bytes in other duplex paths. **Roadmap alignment** This is a tightly scoped reliability and resource-safety improvement. It does not duplicate a roadmap feature. **Additional context** The aggregate ceiling uses a safe 256 MiB default. An invalid override falls back to that default and reports the rejected value. ## What Changed - Add a process-owned aggregate byte ledger for duplex route resource use. - Charge and release route data, broker forward and response bytes, decoder buffers, and readiness replay bytes. - Bound host-to-worker pending writes and standard input transport bytes. - Add a separate decoder cap for the sandbox_process path. - Make invalid aggregate-ceiling overrides fall back to the safe default without host startup failure. - Add adapter-utils and server tests for charging, release, rejection, cleanup, and many-route aggregate limits. ## Verification - pnpm --filter @paperclipai/adapter-utils exec tsc --noEmit - pnpm --filter @paperclipai/server exec tsc --noEmit - Run the focused adapter-utils duplex ledger and execution-target tests. - Run the server aggregate-ledger route test. - Confirm all required pull request checks pass on this branch. ## Risks The ledger touches several duplex buffer paths. A missed release could reduce later capacity until process restart. The tests cover charge, release, rejection, cleanup, and route aggregation. The change uses a safe default when configuration input is invalid. ## Model Used OpenAI GPT-5 Codex. The runtime model ID and context window are not exposed to this task. The model used tool calls, shell commands, and code review workflow support. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with Fixes/Closes/Refs OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub issue references) - [x] My branch name describes the change and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/duplex-aggregate-byte-ledger.test.ts | 221 ++++++ .../src/duplex-aggregate-byte-ledger.ts | 400 +++++++++++ .../duplex-bridge-broker-byte-ledger.test.ts | 317 +++++++++ .../adapter-utils/src/duplex-bridge-broker.ts | 269 +++++++- .../src/duplex-frame-codec.test.ts | 112 ++++ .../adapter-utils/src/duplex-frame-codec.ts | 95 ++- .../adapter-utils/src/duplex-telemetry.ts | 40 +- .../src/execution-target-sandbox.test.ts | 634 +++++++++++++++++- .../adapter-utils/src/execution-target.ts | 326 ++++++++- .../src/sandbox-callback-bridge.ts | 54 +- packages/shared/src/telemetry/README.md | 18 + .../duplex-aggregate-ceiling-env.test.ts | 70 ++ ...nvironment-execution-target-duplex.test.ts | 49 ++ .../fixtures/plugin-worker-duplex-channel.cjs | 37 +- ...-worker-manager-duplex-byte-ledger.test.ts | 348 ++++++++++ ...anager-duplex-pending-write-ledger.test.ts | 251 +++++++ ...-manager-duplex-stdin-write-ledger.test.ts | 297 ++++++++ server/src/duplex-aggregate-ceiling-env.ts | 32 + server/src/index.ts | 59 +- .../services/environment-execution-target.ts | 9 + .../services/environment-run-orchestrator.ts | 10 + server/src/services/heartbeat.ts | 10 + server/src/services/plugin-worker-manager.ts | 538 +++++++++++++-- 23 files changed, 4087 insertions(+), 109 deletions(-) create mode 100644 packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts create mode 100644 packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts create mode 100644 packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts create mode 100644 server/src/__tests__/duplex-aggregate-ceiling-env.test.ts create mode 100644 server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts create mode 100644 server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts create mode 100644 server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts create mode 100644 server/src/duplex-aggregate-ceiling-env.ts diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts new file mode 100644 index 00000000000..790100509e8 --- /dev/null +++ b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + DUPLEX_AGGREGATE_TOKEN_OWNERS, + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + DuplexAggregateByteLedger, + MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX, + assertDuplexAggregateCeilingBytes, + resolveDuplexAggregateCeilingBytes, + type DuplexAggregateByteLedgerTelemetry, +} from "./duplex-aggregate-byte-ledger.js"; + +// A telemetry surface that counts each fixed record, so a test asserts the exact +// gauge value and the exact counter counts. +function createCountingTelemetry(): DuplexAggregateByteLedgerTelemetry & { + gauge: number; + rejections: number; + underflows: number; +} { + const state = { gauge: 0, rejections: 0, underflows: 0 }; + return { + ...state, + setBytesInUse(bytes: number) { + this.gauge = bytes; + }, + recordReservationRejection() { + this.rejections += 1; + }, + recordAccountingUnderflow() { + this.underflows += 1; + }, + }; +} + +describe("DuplexAggregateByteLedger", () => { + it("reserves under the ceiling and tracks the gauge and the live-token count", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + const a = ledger.reserve("buffered_chunk", 400); + const b = ledger.reserve("buffered_chunk", 500); + + expect(a).not.toBeNull(); + expect(b).not.toBeNull(); + expect(ledger.bytesInUse).toBe(900); + expect(ledger.liveTokenCount).toBe(2); + expect(telemetry.gauge).toBe(900); + expect(a?.bytes).toBe(400); + expect(a?.owner).toBe("buffered_chunk"); + expect(a?.state).toBe("held"); + }); + + it("rejects a one-byte-over reservation, retains nothing, and increments the rejection counter", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + ledger.reserve("buffered_chunk", 1000); + const over = ledger.reserve("buffered_chunk", 1); + + expect(over).toBeNull(); + expect(ledger.bytesInUse).toBe(1000); + expect(ledger.liveTokenCount).toBe(1); + expect(telemetry.rejections).toBe(1); + // The public rejection marker is the fixed string the caller reports. + expect(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED).toBe("DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED"); + }); + + it("transfers ownership without a decrement or a re-reserve, so no admission gap opens", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 100, telemetry }); + + // Reserve the whole ceiling as a pre-bind event, then transfer to buffered. + const token = ledger.reserve("pre_bind_event", 100); + expect(token).not.toBeNull(); + // A second reservation cannot fit while the token stays held. + expect(ledger.reserve("buffered_chunk", 1)).toBeNull(); + + ledger.transfer(token!, "buffered_chunk"); + + // The transfer changed only the owner. The gauge never moved, so no window + // opened where the bytes were released and re-admitted. + expect(ledger.bytesInUse).toBe(100); + expect(ledger.liveTokenCount).toBe(1); + expect(token?.owner).toBe("buffered_chunk"); + expect(telemetry.gauge).toBe(100); + // No reservation fit during the transfer, so no rejection or underflow fired + // beyond the one rejection above. + expect(telemetry.rejections).toBe(1); + expect(telemetry.underflows).toBe(0); + }); + + it("releases a token one time and decrements the gauge", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + const token = ledger.reserve("response_body", 250); + expect(token).not.toBeNull(); + ledger.release(token!); + + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(token?.state).toBe("released"); + expect(telemetry.gauge).toBe(0); + expect(telemetry.underflows).toBe(0); + }); + + it("records the underflow counter on a second release and does not clamp the gauge again", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + const kept = ledger.reserve("request_frame", 300); + const doubled = ledger.reserve("request_frame", 200); + expect(kept).not.toBeNull(); + expect(doubled).not.toBeNull(); + ledger.release(doubled!); + expect(ledger.bytesInUse).toBe(300); + + // A forced second release of the same token is an accounting defect. + ledger.release(doubled!); + + expect(telemetry.underflows).toBe(1); + // The gauge did not drop a second time, so the defect stays visible. + expect(ledger.bytesInUse).toBe(300); + expect(ledger.liveTokenCount).toBe(1); + }); + + it("records the underflow counter on a transfer of a released token", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + const token = ledger.reserve("pre_bind_event", 100); + ledger.release(token!); + + ledger.transfer(token!, "buffered_chunk"); + + expect(telemetry.underflows).toBe(1); + expect(ledger.bytesInUse).toBe(0); + }); + + it("throws on a non-integer or negative reservation, so a programming error fails loud", () => { + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 }); + expect(() => ledger.reserve("buffered_chunk", -1)).toThrow(); + expect(() => ledger.reserve("buffered_chunk", 1.5)).toThrow(); + expect(() => ledger.reserve("buffered_chunk", Number.NaN)).toThrow(); + }); + + it("admits a zero-byte reservation as a valid held token", () => { + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 }); + const token = ledger.reserve("seen_request_id", 0); + expect(token).not.toBeNull(); + expect(token?.bytes).toBe(0); + expect(ledger.liveTokenCount).toBe(1); + }); + + it("exposes the closed owner-label set", () => { + expect([...DUPLEX_AGGREGATE_TOKEN_OWNERS]).toEqual([ + "pre_bind_event", + "buffered_chunk", + "terminal_buffered", + "request_frame", + "request_payload", + "response_body", + "seen_request_id", + "decoder_buffer", + "readiness_buffer", + "readiness_replay", + "pending_write", + "stdin_write", + ]); + }); +}); + +describe("aggregate ceiling configuration", () => { + it("uses the default when the override is absent", () => { + expect(resolveDuplexAggregateCeilingBytes()).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + expect(resolveDuplexAggregateCeilingBytes(null)).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + expect(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES).toBe(256 * 1024 * 1024); + }); + + it("accepts a valid positive override under the process allocation minus the reserve", () => { + expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024)).toBe(64 * 1024 * 1024); + const maxCeiling = + MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; + expect(resolveDuplexAggregateCeilingBytes(maxCeiling)).toBe(maxCeiling); + }); + + it("rejects an invalid override to the safe default and reports each rejection", () => { + const overMaximum = MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX; + const invalidOverrides = [0, -1, 1.5, Number.POSITIVE_INFINITY, Number.NaN, overMaximum]; + for (const invalid of invalidOverrides) { + const rejected: number[] = []; + // An invalid present override returns the safe default. It never throws, so a + // single bad environment value never fails host startup. + expect(resolveDuplexAggregateCeilingBytes(invalid, (value) => rejected.push(value))).toBe( + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + ); + // The resolver reports the rejection one time with the rejected numeric value. + expect(rejected).toEqual([invalid]); + } + }); + + it("does not report a rejection for a valid or a missing override", () => { + const rejected: number[] = []; + const reporter = (value: number): void => { + rejected.push(value); + }; + expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024, reporter)).toBe(64 * 1024 * 1024); + expect(resolveDuplexAggregateCeilingBytes(undefined, reporter)).toBe( + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + ); + expect(resolveDuplexAggregateCeilingBytes(null, reporter)).toBe( + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + ); + expect(rejected).toEqual([]); + }); + + it("still throws for a direct ceiling assertion on an unsafe integer", () => { + // The ledger constructor asserts its ceiling and must fail loud on a + // programming error, so `assertDuplexAggregateCeilingBytes` keeps throwing. + expect(() => assertDuplexAggregateCeilingBytes(Number.MAX_SAFE_INTEGER + 2)).toThrow(); + }); +}); diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts new file mode 100644 index 00000000000..e7f7aec3c16 --- /dev/null +++ b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts @@ -0,0 +1,400 @@ +/** + * The process-owned aggregate byte ledger for the sandbox duplex channel. + * + * The route-count controller bounds how many concurrent duplex routes stay live. + * It does not bound the aggregate bytes those routes retain. At the maximum route + * count the per-route byte bounds multiply to many gigabytes of retained bytes. + * This ledger closes that gap. One ledger per host server process owns a single + * byte ceiling. Every host-side retention site reserves its exact retained bytes + * against this ledger before it allocates, so the aggregate retained bytes across + * all live routes never pass the ceiling. + * + * The ledger is fail-closed. A reservation that would pass the ceiling returns no + * token. The caller retains nothing and reports the fixed rejection marker + * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. + * + * Ownership is transfer-only. A token holds an immutable byte amount, an owner + * label from a closed enum, and a one-way state: `held` then `released`. The + * ledger moves a token from one representation to the next with + * {@link DuplexAggregateByteLedger.transfer}, which changes only the owner label. + * It never decrements and re-reserves, so no admission gap opens between two + * representations of the same retained bytes. + * + * The ledger holds every live token in one owner registry. Registry removal and + * the release of a token run in one synchronous critical section, before any + * `await` or external callback. A second release of the same token is an + * accounting defect. The ledger records the fixed underflow counter and does not + * clamp the gauge down a second time, so a real defect stays visible. + * + * The ledger carries no payload data and no arbitrary labels. It exposes only the + * byte gauge, the live-token count, and the closed owner enum. + */ + +import { + DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, + DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, + DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, +} from "./duplex-telemetry.js"; + +/** + * The public rejection marker. A retention site that cannot reserve its bytes + * reports this fixed string. It carries no payload, no route, and no raw value. + */ +export const DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED = "DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED"; + +/** + * The default aggregate ceiling, in bytes: 256 MiB. This is one eighth of the + * documented minimum host process memory allocation for the sandbox duplex + * transport ({@link MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX}, 2 GiB). The + * remaining seven eighths cover the Node runtime, ordinary server work, parser + * and transient headroom, and allocator or garbage-collector variance. + */ +export const DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES = 256 * 1024 * 1024; + +/** + * The documented minimum host process memory allocation to enable the sandbox + * duplex transport, in bytes: 2 GiB. An operator override of the aggregate + * ceiling must not pass this value minus the documented reserve. + */ +export const MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX = 2 * 1024 * 1024 * 1024; + +/** + * The closed set of owner labels a reservation token can carry. Each label names + * one host-side representation that retains bytes. The ledger accepts no label + * outside this set, so no arbitrary label ever reaches a token. + * + * - `pre_bind_event`: a held pre-bind duplex event, before route bind. + * - `buffered_chunk`: a buffered data chunk retained for a late listener. + * - `terminal_buffered`: a buffered chunk moved to the terminal registry at + * route terminalization, held for a late listener drain. + * - `request_frame`: a bounded raw request frame the broker retains at dispatch. + * - `request_payload`: a normalized request payload the broker retains at dispatch. + * - `response_body`: a response-body chunk or replacement buffer the reader retains. + * - `seen_request_id`: one entry of the broker no-replay request-id set. + * - `decoder_buffer`: the raw partial-frame bytes the host frame decoder retains + * between chunks, plus the peak replacement buffer it allocates on concat. + * - `readiness_buffer`: the raw untrusted bytes the readiness gate retains before + * the READY frame completes, before the broker binds and the decoder takes over. + * - `readiness_replay`: the post-READY suffix bytes the readiness gate holds in the + * pending replay buffer, from the READY accept until the broker binds and the + * frame decoder charges its own retention. This is a separate retention from + * `readiness_buffer`: the gate drops the whole pre-READY buffer on READY, then + * charges only the retained suffix and each later pre-bind chunk under this owner. + * - `pending_write`: the raw host-to-worker write payload a pending duplex write + * RPC retains, from the enqueue seam until the RPC settles. + * - `stdin_write`: the serialized host-to-worker frame the child-stdin transport + * buffer retains, from the write until the stream flushes the chunk, the stream + * errors, the stream closes, or the worker exits. This is a separate retention + * from `pending_write`: the RPC holds the raw payload while the transport buffer + * holds the larger serialized frame, so the two tokens cover the peak of both. + */ +export const DUPLEX_AGGREGATE_TOKEN_OWNERS = [ + "pre_bind_event", + "buffered_chunk", + "terminal_buffered", + "request_frame", + "request_payload", + "response_body", + "seen_request_id", + "decoder_buffer", + "readiness_buffer", + "readiness_replay", + "pending_write", + "stdin_write", +] as const; + +/** One owner label from the closed {@link DUPLEX_AGGREGATE_TOKEN_OWNERS} set. */ +export type DuplexAggregateTokenOwner = (typeof DUPLEX_AGGREGATE_TOKEN_OWNERS)[number]; + +/** The one-way lifecycle state of a reservation token. */ +export type DuplexReservationTokenState = "held" | "released"; + +/** + * An opaque reservation token. The byte amount is immutable. The owner label + * moves through the closed enum by transfer only. The state moves one way, from + * `held` to `released`. A caller reads these fields but never mutates them; only + * the owning ledger changes the owner and the state. + */ +export interface ReservationToken { + /** The immutable exact byte amount this token reserves. */ + readonly bytes: number; + /** The current owner label. The ledger changes it only through `transfer`. */ + readonly owner: DuplexAggregateTokenOwner; + /** The one-way lifecycle state. The ledger sets it to `released` one time. */ + readonly state: DuplexReservationTokenState; +} + +/** + * The internal mutable token. The ledger owns it and returns it typed as the + * read-only {@link ReservationToken}. Only code in this module changes the owner + * or the state. + */ +class ReservationTokenImpl implements ReservationToken { + readonly bytes: number; + owner: DuplexAggregateTokenOwner; + state: DuplexReservationTokenState = "held"; + + constructor(bytes: number, owner: DuplexAggregateTokenOwner) { + this.bytes = bytes; + this.owner = owner; + } +} + +/** + * The fixed telemetry surface for the aggregate byte ledger. The host binds it to + * the closed duplex telemetry contract. Each record uses only fixed constant + * dimensions. No company, agent, run, route, URL, header, request id, token, byte + * payload, or raw provider value ever reaches a record. The default is a no-op. + */ +export interface DuplexAggregateByteLedgerTelemetry { + /** Set the aggregate-bytes-in-use gauge to the current value. */ + setBytesInUse(bytes: number): void; + /** Increment the reservation-rejection counter one time. */ + recordReservationRejection(): void; + /** Increment the accounting-underflow counter one time. */ + recordAccountingUnderflow(): void; +} + +/** A no-op telemetry surface. Every method does nothing, so the ledger stays inert. */ +export const NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY: DuplexAggregateByteLedgerTelemetry = { + setBytesInUse() {}, + recordReservationRejection() {}, + recordAccountingUnderflow() {}, +}; + +/** + * The low-level process metric sink the host binds. It carries the fixed metric + * name and the value only, so no dynamic dimension reaches a sink. The host maps + * the gauge and the two counters to the process metric pipeline and to a warn log + * for a defect. + */ +export interface DuplexAggregateByteLedgerMetricSink { + /** Set a named process gauge to a value. */ + setGauge(name: string, value: number): void; + /** Increment a named process counter one time. */ + incrementCounter(name: string): void; +} + +/** + * Build a {@link DuplexAggregateByteLedgerTelemetry} that maps each ledger event + * to the fixed metric name on the process metric sink. The gauge uses + * {@link DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE}. The rejection counter uses + * {@link DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL}. The defect + * counter uses {@link DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL}. + * Each call sits in an error swallow, so a sink failure never breaks the reserve + * or the release path. + */ +export function createDuplexAggregateByteLedgerTelemetry( + sink: DuplexAggregateByteLedgerMetricSink, +): DuplexAggregateByteLedgerTelemetry { + return { + setBytesInUse(bytes: number): void { + try { + sink.setGauge(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, bytes); + } catch { + // A telemetry failure never breaks the reserve or the release path. + } + }, + recordReservationRejection(): void { + try { + sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL); + } catch { + // A telemetry failure never breaks the reserve path. + } + }, + recordAccountingUnderflow(): void { + try { + sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL); + } catch { + // A telemetry failure never breaks the release path. + } + }, + }; +} + +/** The options for {@link DuplexAggregateByteLedger}. */ +export interface DuplexAggregateByteLedgerOptions { + /** The aggregate ceiling, in bytes. It must be a positive safe integer. */ + ceilingBytes: number; + /** The fixed telemetry surface. The default is the no-op surface. */ + telemetry?: DuplexAggregateByteLedgerTelemetry; +} + +/** + * Assert an aggregate ceiling byte value. The value must be a positive safe + * integer that is no larger than the documented process allocation minus the + * documented reserve. The function throws on any other value, so an invalid + * ceiling fails startup instead of silently disabling the ledger. + * + * The reserve equals the documented process allocation minus the default + * ceiling ({@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}), so the maximum + * accepted ceiling is exactly the documented process allocation minus that + * reserve. + */ +export function assertDuplexAggregateCeilingBytes(value: number): void { + const maxCeiling = maxDuplexAggregateCeilingBytes(); + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error( + `The duplex aggregate byte ceiling must be a positive safe integer; got ${String(value)}.`, + ); + } + if (value > maxCeiling) { + throw new Error( + `The duplex aggregate byte ceiling ${value} must not pass the documented process allocation minus the reserve (${maxCeiling}).`, + ); + } +} + +/** + * The maximum accepted aggregate ceiling, in bytes: the documented process + * allocation minus the documented reserve. Both {@link assertDuplexAggregateCeilingBytes} + * and {@link isValidDuplexAggregateCeilingBytes} read this one bound. + */ +function maxDuplexAggregateCeilingBytes(): number { + return MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; +} + +/** + * Report whether `value` is a valid aggregate ceiling. A valid ceiling is a + * positive safe integer that is no larger than {@link maxDuplexAggregateCeilingBytes}. + * This is the non-throwing form of {@link assertDuplexAggregateCeilingBytes}. The + * override resolver uses it to reject an invalid override without a throw. + */ +export function isValidDuplexAggregateCeilingBytes(value: number): boolean { + return Number.isSafeInteger(value) && value >= 1 && value <= maxDuplexAggregateCeilingBytes(); +} + +/** + * A reporter the host binds to receive one invalid-override rejection signal. + * {@link resolveDuplexAggregateCeilingBytes} calls it one time when it rejects a + * present invalid override and falls back to the safe default. The signal carries + * only the rejected numeric value. It carries no route, company, run, or payload + * value. + */ +export type DuplexAggregateCeilingOverrideRejectionReporter = (rejectedValue: number) => void; + +/** + * Resolve an aggregate ceiling from an optional operator override. A missing or + * `null` override uses {@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}. A present + * valid value passes through unchanged. + * + * A present invalid override does not fail host startup. The host process is + * multi-tenant, so one invalid environment value must not brick the whole host. + * The resolver rejects the invalid override, reports it through the optional + * `onRejectedOverride` reporter, and returns the conservative safe default. This + * fails loud without failing closed on availability. + */ +export function resolveDuplexAggregateCeilingBytes( + override?: number | null, + onRejectedOverride?: DuplexAggregateCeilingOverrideRejectionReporter, +): number { + if (override === undefined || override === null) { + return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; + } + if (!isValidDuplexAggregateCeilingBytes(override)) { + onRejectedOverride?.(override); + return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; + } + return override; +} + +/** + * The process-owned aggregate byte ledger. Create one per host server process. + * Inject the same object into every host-side retention site, so one shared gauge + * bounds the aggregate retained bytes across all live duplex routes. + */ +export class DuplexAggregateByteLedger { + private readonly ceiling: number; + private readonly telemetry: DuplexAggregateByteLedgerTelemetry; + private used = 0; + /** The owner registry. It holds every live token. */ + private readonly liveTokens = new Set(); + + constructor(options: DuplexAggregateByteLedgerOptions) { + assertDuplexAggregateCeilingBytes(options.ceilingBytes); + this.ceiling = options.ceilingBytes; + this.telemetry = options.telemetry ?? NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY; + } + + /** The configured aggregate ceiling, in bytes. */ + get ceilingBytes(): number { + return this.ceiling; + } + + /** The current aggregate bytes in use across every live token. */ + get bytesInUse(): number { + return this.used; + } + + /** The current number of live tokens in the owner registry. */ + get liveTokenCount(): number { + return this.liveTokens.size; + } + + /** + * Reserve `bytes` for `owner`. Check and increment the gauge synchronously + * before the caller allocates. Return a held token when the reservation fits + * under the ceiling. Return `null` when `used + bytes` would pass the ceiling; + * the caller retains nothing and reports + * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. + * + * The byte amount must be a non-negative safe integer. A caller that passes any + * other value made a programming error, so the ledger throws and fails loud. + */ + reserve(owner: DuplexAggregateTokenOwner, bytes: number): ReservationToken | null { + if (!Number.isSafeInteger(bytes) || bytes < 0) { + throw new Error( + `A duplex byte reservation must be a non-negative safe integer; got ${String(bytes)}.`, + ); + } + if (this.used + bytes > this.ceiling) { + this.telemetry.recordReservationRejection(); + return null; + } + this.used += bytes; + const token = new ReservationTokenImpl(bytes, owner); + this.liveTokens.add(token); + this.telemetry.setBytesInUse(this.used); + return token; + } + + /** + * Transfer a held token to a new owner label. Change only the owner. Never + * decrement and never re-reserve. This is the atomic pre-bind handoff: it moves + * the same reserved bytes from one representation to the next with no admission + * gap. + * + * A transfer of a token the ledger does not hold, or of an already-released + * token, is an accounting defect. The ledger records the fixed underflow + * counter and leaves the gauge unchanged. + */ + transfer(token: ReservationToken, owner: DuplexAggregateTokenOwner): void { + const impl = token as ReservationTokenImpl; + if (impl.state !== "held" || !this.liveTokens.has(impl)) { + this.telemetry.recordAccountingUnderflow(); + return; + } + impl.owner = owner; + } + + /** + * Release a held token. Remove it from the owner registry, mark it `released`, + * and decrement the gauge, all in one synchronous critical section before any + * `await` or external callback. A token releases one time. + * + * A second release of the same token, or a release of a token the ledger does + * not hold, is an accounting defect. The ledger records the fixed underflow + * counter and does not clamp the gauge a second time, so a real defect stays + * visible on the counter. + */ + release(token: ReservationToken): void { + const impl = token as ReservationTokenImpl; + if (impl.state === "released" || !this.liveTokens.delete(impl)) { + this.telemetry.recordAccountingUnderflow(); + return; + } + impl.state = "released"; + this.used -= impl.bytes; + this.telemetry.setBytesInUse(this.used); + } +} diff --git a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts new file mode 100644 index 00000000000..78780c9ff59 --- /dev/null +++ b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts @@ -0,0 +1,317 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { + createDuplexBridgeBroker, + type DuplexBridgeBroker, + type DuplexBrokerForwardResult, +} from "./duplex-bridge-broker.js"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + DuplexAggregateByteLedger, +} from "./duplex-aggregate-byte-ledger.js"; +import { + DUPLEX_FRAME_VERSION, + DuplexFrameDecoder, + encodeDuplexFrame, + type DuplexRequestFrame, + type DuplexResponseFrame, +} from "./duplex-frame-codec.js"; +import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; + +/** + * Regression harness for the broker aggregate byte ledger charging. + * + * The harness feeds request frames straight into the broker through an in-memory + * channel, the same as a provider that controls the transport. A test controls + * when each forward settles, so it can assert the ledger charge while the forward + * is still in flight and after it settles. + */ + +/** One pending forward a test settles by hand. */ +interface PendingForward { + id: string; + resolve: (result: DuplexBrokerForwardResult) => void; + reject: (error: Error) => void; + settled: boolean; +} + +/** The in-memory channel plus the levers a test uses to drive the broker. */ +interface FakeChannelHarness { + channel: CommandManagedDuplexChannel; + feed: (frame: DuplexRequestFrame) => void; + exit: () => void; + responses: DuplexResponseFrame[]; + forwards: PendingForward[]; + resolveForward: (id: string, body?: string) => void; +} + +/** Build one valid request frame. */ +function requestFrame(id: string, method = "POST"): DuplexRequestFrame { + return { + version: DUPLEX_FRAME_VERSION, + type: "request", + id, + method, + path: `/api/issues/${id}`, + query: "", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ id }), + }; +} + +function createFakeChannelHarness(): FakeChannelHarness { + const responses: DuplexResponseFrame[] = []; + const forwards: PendingForward[] = []; + const writtenDecoder = new DuplexFrameDecoder(); + let dataListener: ((chunk: string) => void) | null = null; + let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; + + const channel: CommandManagedDuplexChannel = { + write: (data) => { + for (const result of writtenDecoder.push(data)) { + if (result.ok && result.frame.type === "response") responses.push(result.frame); + } + }, + onData: (listener) => { + dataListener = listener; + }, + onExit: (listener) => { + exitListener = listener; + }, + stop: () => undefined, + close: () => Promise.resolve(), + }; + + return { + channel, + feed: (frame) => { + if (!dataListener) throw new Error("The broker did not bind the data listener."); + dataListener(encodeDuplexFrame(frame)); + }, + exit: () => { + if (!exitListener) throw new Error("The broker did not bind the exit listener."); + exitListener({ exitCode: 0 }); + }, + responses, + forwards, + resolveForward: (id, body = JSON.stringify({ ok: true })) => { + const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled); + if (!forward) throw new Error(`No unsettled forward for id ${id}.`); + forward.settled = true; + forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body }); + }, + }; +} + +/** A forward handler that hands each call to the harness and never auto-resolves. */ +function controllableForward(harness: FakeChannelHarness) { + return (request: DuplexRequestFrame): Promise => + new Promise((resolve, reject) => { + harness.forwards.push({ id: request.id, resolve, reject, settled: false }); + }); +} + +/** Wait for the microtask queue to drain, so a settled promise runs its handlers. */ +async function flush(): Promise { + await Promise.resolve(); + await Promise.resolve(); +} + +describe("duplex bridge broker aggregate byte ledger", () => { + const brokers: DuplexBridgeBroker[] = []; + + afterEach(async () => { + while (brokers.length > 0) { + const broker = brokers.pop(); + if (broker) await broker.close(); + } + }); + + it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => { + const harness = createFakeChannelHarness(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + // The dispatch retains three tokens: the raw frame, the normalized payload, + // and the no-replay set entry. + expect(ledger.liveTokenCount).toBe(3); + expect(ledger.bytesInUse).toBeGreaterThan(0); + const chargedInFlight = ledger.bytesInUse; + + harness.resolveForward("req-1"); + await flush(); + + // The forward settled. The finally owner released the request-frame and the + // request-payload tokens. The seen-id token stays charged for the channel + // lifetime, so exactly one token remains. + expect(ledger.liveTokenCount).toBe(1); + expect(ledger.bytesInUse).toBeGreaterThan(0); + expect(ledger.bytesInUse).toBeLessThan(chargedInFlight); + + // A close releases the seen-id token, so the ledger returns to zero. + await broker.close(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => { + const harness = createFakeChannelHarness(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + // Squeeze the nested budgets so the response timer fires quickly while the + // forward stays unsettled. + budgets: { forwardTimeoutMs: 5, responseBudgetMs: 10, gatewayWaitMs: 20 }, + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1", "GET")); + expect(ledger.liveTokenCount).toBe(3); + + // Wait past the response budget so the backstop answers the gateway while the + // forward is still in flight. + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(harness.responses.some((frame) => frame.id === "req-1")).toBe(true); + // The gateway got its answer, but the forward has not settled. The request + // tokens and the seen-id token stay charged: nothing released yet. + expect(ledger.liveTokenCount).toBe(3); + + // Settle the orphaned forward. Its finally owner now releases the two request + // tokens; the seen-id token still stays for the channel lifetime. + harness.resolveForward("req-1"); + await flush(); + expect(ledger.liveTokenCount).toBe(1); + + await broker.close(); + expect(ledger.bytesInUse).toBe(0); + }); + + it("refuses a one-byte-over dispatch with the fixed marker and retains nothing", async () => { + const harness = createFakeChannelHarness(); + // Measure the exact per-request charge with a throwaway broker, so the test + // can size a ceiling one byte below it and force the reservation to fail. + const probeHarness = createFakeChannelHarness(); + const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const measuringBroker = createDuplexBridgeBroker({ + channel: probeHarness.channel, + forwardRequest: controllableForward(probeHarness), + duplexAggregateByteLedger: measured, + }); + measuringBroker.start(); + probeHarness.feed(requestFrame("req-1")); + const perRequestBytes = measured.bytesInUse; + await measuringBroker.close(); + + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 }); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + await flush(); + + // The broker retained nothing: no token, no forward, no seen id. + expect(ledger.liveTokenCount).toBe(0); + expect(ledger.bytesInUse).toBe(0); + expect(harness.forwards.length).toBe(0); + // The refusal carries the fixed marker and is a bounded terminal response. + const refusal = harness.responses.find((frame) => frame.id === "req-1"); + expect(refusal).toBeTruthy(); + expect(refusal?.status).toBe(503); + expect(refusal?.body).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + + // The refusal did not retain the id, so a resend after pressure eases is + // admitted: raise the ceiling and re-feed. + const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const roomyHarness = createFakeChannelHarness(); + const roomyBroker = createDuplexBridgeBroker({ + channel: roomyHarness.channel, + forwardRequest: controllableForward(roomyHarness), + duplexAggregateByteLedger: roomyLedger, + }); + brokers.push(roomyBroker); + roomyBroker.start(); + roomyHarness.feed(requestFrame("req-1")); + expect(roomyHarness.forwards.length).toBe(1); + }); + + it("releases every retained token on a terminal channel loss", async () => { + const harness = createFakeChannelHarness(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + harness.feed(requestFrame("req-2")); + expect(ledger.liveTokenCount).toBe(6); + + // The channel exits mid-flight. The broker latches the loss, transfers the + // in-flight forwards to the orphan registry, and releases the seen-id tokens. + harness.exit(); + expect(broker.runDisposition.failed).toBe(true); + // The seen-id tokens released at loss; the two request tokens per forward stay + // charged until each aborted forward settles. + expect(ledger.liveTokenCount).toBe(4); + + // The aborted forwards settle. Each finally owner releases its two request + // tokens, so the ledger returns to zero. + harness.forwards[0]?.reject(new Error("aborted")); + harness.forwards[1]?.reject(new Error("aborted")); + await flush(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("does not double-release a token when a forward settles after the channel closed", async () => { + const harness = createFakeChannelHarness(); + let underflow = 0; + const ledger = new DuplexAggregateByteLedger({ + ceilingBytes: 1_000_000, + telemetry: { + setBytesInUse() {}, + recordReservationRejection() {}, + recordAccountingUnderflow() { + underflow += 1; + }, + }, + }); + const broker = createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + await broker.close(); + // The close aborted the in-flight forward and orphaned it. The forward now + // settles after the close: its finally owner releases the request tokens once. + harness.forwards[0]?.reject(new Error("aborted")); + await flush(); + + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + // No accounting defect: every token released exactly one time. + expect(underflow).toBe(0); + }); +}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts index 1a902360a9b..b7a04021a15 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.ts @@ -42,6 +42,11 @@ */ import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + type DuplexAggregateByteLedger, + type ReservationToken, +} from "./duplex-aggregate-byte-ledger.js"; import { DEFAULT_MAX_DUPLEX_FRAME_BYTES, DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, @@ -169,6 +174,17 @@ export const DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS = 64; */ export const DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS = 50_000; +/** + * The fixed, documented per-entry allocation the broker charges the aggregate byte + * ledger for one no-replay request-id set entry. The type and the cardinality of + * `seenRequestIds` are bounded: the codec caps each id at + * {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}, and the lifetime limit caps the + * entry count. The broker charges the exact raw id bytes plus this fixed entry + * overhead, so the retained set never grows uncharged. This constant models the + * fixed per-entry cost of the string key and the Set slot, not the id bytes. + */ +export const DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES = 64; + /** The result of one forward call. The broker turns it into one response frame. */ export interface DuplexBrokerForwardResult { status: number; @@ -250,6 +266,17 @@ export interface DuplexBrokerOptions { * raw error rides a span or a counter. The default records nothing. */ telemetry?: DuplexTelemetry; + /** + * The process-owned aggregate byte ledger. The broker reserves the exact retained + * bytes of each dispatched request against it before it retains the frame: the + * raw request frame, the normalized request payload, and the no-replay set entry. + * A reservation that would pass the ceiling makes the broker retain nothing and + * refuse the request with the fixed marker + * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. When the ledger is absent the + * broker charges nothing and behaves as before, so a non-duplex or a legacy path + * stays unchanged. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } /** The broker handle the factory returns. */ @@ -358,6 +385,27 @@ interface PendingRequest { responseTimer: ReturnType; /** The point the broker started to dispatch the request. It sets the span latency. */ dispatchStartMs: number; + /** True once the forward promise settled. The finally owner sets it one time. */ + forwardSettled: boolean; + /** + * Release the request-frame token and the request-payload token exactly one time. + * The single forward-promise finally owner calls it after the forward settles. + * A second call is a no-op, so no token releases twice. + */ + releaseForwardTokens: () => void; +} + +/** + * One orphaned forward. The broker answered the gateway and removed the request + * from `pending`, but the forward promise or its response-body reader had not + * settled. The orphan keeps the request tokens charged and the controller live + * until the forward finally releases them, so the ledger reports nonzero ownership + * until the async work settles. + */ +interface OrphanedForward { + controller: AbortController; + /** Release the request-frame and request-payload tokens exactly one time. */ + releaseForwardTokens: () => void; } /** @@ -382,12 +430,42 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS; const closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS; const now = options.now ?? (() => Date.now()); + + // The process-owned aggregate byte ledger, or `null` when the caller injected + // none. When `null` the broker charges nothing and behaves as before. + const ledger = options.duplexAggregateByteLedger ?? null; // The one frame size bound the broker enforces on both sides. The decoder // rejects an inbound frame over this bound, and the encode guard refuses to // write an outbound frame over it. Encode and decode share one value, so a // frame the broker writes always decodes on the peer. const maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES; - const decoder = new DuplexFrameDecoder({ maxFrameBytes }); + // The decoder charges its retained partial-frame bytes against the same host + // ledger. It receives the ledger object directly, so one gauge bounds the + // decoder buffer with every other host retention site. + const decoder = new DuplexFrameDecoder({ + maxFrameBytes, + ...(ledger ? { aggregateByteLedger: ledger } : {}), + }); + // Release one token exactly one time. A `null` token or an absent ledger is a + // no-op, so the broker never records a false accounting defect. + const releaseToken = (token: ReservationToken | null): void => { + if (ledger && token) ledger.release(token); + }; + // The byte size of the normalized request payload the broker retains across the + // forward. It counts the exact retained scalar and header bytes, so the charge + // matches the retained bytes and never a parsed object graph. + const requestPayloadBytes = (frame: DuplexRequestFrame): number => { + let bytes = + Buffer.byteLength(frame.id, "utf8") + + Buffer.byteLength(frame.method, "utf8") + + Buffer.byteLength(frame.path, "utf8") + + Buffer.byteLength(frame.query, "utf8") + + Buffer.byteLength(frame.body, "utf8"); + for (const [key, value] of Object.entries(frame.headers)) { + bytes += Buffer.byteLength(key, "utf8") + Buffer.byteLength(value, "utf8"); + } + return bytes; + }; let state: DuplexBrokerState = "opening"; let stopped = false; @@ -432,7 +510,24 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // The ids the broker already dispatched. The broker forwards one id one time, // so a repeated frame never reaches the API twice. const seenRequestIds = new Set(); + // The aggregate-ledger tokens for the no-replay set entries. The broker holds + // one token per live set entry and releases every token one time at terminal + // teardown, so the retained set never leaves bytes charged after the channel + // ends. + const seenRequestIdTokens = new Set(); const pending = new Map(); + // The forwards that answered the gateway but whose async work has not settled. + // The broker keeps their tokens charged and their controller live until each + // forward finally releases its tokens, so the ledger reports the real ownership. + const orphanedForwards = new Map(); + + // Release every no-replay set-entry token one time and clear the set. The + // ledger release is one way, and the cleared set stops any second release, so + // this helper is safe to call more than one time at terminal teardown. + const releaseSeenRequestIdTokens = (): void => { + for (const token of seenRequestIdTokens) releaseToken(token); + seenRequestIdTokens.clear(); + }; const setState = (next: DuplexBrokerState): void => { if (state === next) return; @@ -448,12 +543,27 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr }; const clearPending = (): void => { - for (const entry of pending.values()) { + for (const [id, entry] of pending) { clearTimeout(entry.forwardTimer); clearTimeout(entry.responseTimer); entry.controller.abort(new Error("Duplex broker stopped.")); + // A forward that has not settled still owns its request tokens. Transfer it + // to the orphan registry, so its tokens stay charged until the forward + // finally releases them. The abort only asks the forward to stop; it never + // releases a token by itself. + if (!entry.forwardSettled) { + orphanedForwards.set(id, { + controller: entry.controller, + releaseForwardTokens: entry.releaseForwardTokens, + }); + } } pending.clear(); + // Ask every already-orphaned forward to stop as well. The forward-promise + // finally owner releases each orphan token when the forward settles. + for (const orphan of orphanedForwards.values()) { + orphan.controller.abort(new Error("Duplex broker stopped.")); + } }; const recordLoss = (reason: DuplexBrokerLossReason, message: string): void => { @@ -470,6 +580,8 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr stopped = true; clearHeartbeat(); clearPending(); + releaseSeenRequestIdTokens(); + decoder.dispose(); if (state !== "closing") setState("closed"); return; } @@ -493,6 +605,8 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } clearHeartbeat(); clearPending(); + releaseSeenRequestIdTokens(); + decoder.dispose(); lossRecord = { reason, message, atMs: now() }; setState("lost"); // Log the internal reason only. The broker never writes the raw provider @@ -600,6 +714,17 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr clearTimeout(entry.forwardTimer); clearTimeout(entry.responseTimer); pending.delete(id); + // The response timer, an abort, or a broker loss can answer the gateway before + // the forward promise settles. When the forward has not settled, transfer it to + // the orphan registry so its request tokens stay charged until the forward + // finally releases them. A settled forward already released or will release + // through its finally owner, so it needs no orphan record. + if (!entry.forwardSettled) { + orphanedForwards.set(id, { + controller: entry.controller, + releaseForwardTokens: entry.releaseForwardTokens, + }); + } // Do not write on a lost or closed channel. The gateway answers its own // outstanding request on loss, so a late write would go to a dead channel. if (state !== "open") return; @@ -662,6 +787,34 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr writeFrame(frame); }; + const respondAggregateExceeded = (id: string): void => { + // Answer a request the aggregate byte ledger refused with a bounded terminal + // response. The broker reserved nothing, added no id to the seen set, and made + // no controller, timer, or forward, so the host API stays untouched. The + // response carries the fixed marker only; it holds no route, query, body, or + // token. The `unavailable` outcome tells the gateway this is not a delivered + // host response. The refusal is retryable, because the broker did not retain + // the id: the aggregate pressure can ease, and a resend can then get through. + if (state !== "open") return; + const frame: DuplexResponseFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: 503, + headers: { + "content-type": "application/json", + "x-paperclip-bridge-outcome": "unavailable", + }, + body: JSON.stringify({ + error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + outcome: "unavailable", + retryable: true, + }), + outcome: "unavailable", + }; + writeFrame(frame); + }; + const dispatch = (frame: DuplexRequestFrame): void => { // Dispatch only while open. After loss or close the broker forwards nothing. if (state !== "open") return; @@ -699,7 +852,54 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr respondSaturated(frame.id, true); return; } + // Reserve the exact retained bytes against the aggregate ledger before the + // broker retains anything. It reserves three tokens in order: the raw request + // frame, the normalized request payload, and the no-replay set entry. A + // reservation that would pass the ceiling makes the broker retain nothing, + // release the tokens it already took, and refuse the request with the fixed + // marker. The broker adds no id to the seen set and makes no controller, timer, + // or forward, so the host API stays untouched. When the ledger is absent all + // tokens stay `null` and the broker behaves as before. + let requestFrameToken: ReservationToken | null = null; + let requestPayloadToken: ReservationToken | null = null; + let seenRequestIdToken: ReservationToken | null = null; + if (ledger) { + const rawFrameBytes = Buffer.byteLength(encodeDuplexFrame(frame), "utf8"); + requestFrameToken = ledger.reserve("request_frame", rawFrameBytes); + if (!requestFrameToken) { + respondAggregateExceeded(frame.id); + return; + } + requestPayloadToken = ledger.reserve("request_payload", requestPayloadBytes(frame)); + if (!requestPayloadToken) { + ledger.release(requestFrameToken); + respondAggregateExceeded(frame.id); + return; + } + const seenEntryBytes = + Buffer.byteLength(frame.id, "utf8") + DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES; + seenRequestIdToken = ledger.reserve("seen_request_id", seenEntryBytes); + if (!seenRequestIdToken) { + ledger.release(requestFrameToken); + ledger.release(requestPayloadToken); + respondAggregateExceeded(frame.id); + return; + } + } seenRequestIds.add(frame.id); + if (seenRequestIdToken) seenRequestIdTokens.add(seenRequestIdToken); + + // Release the request-frame and the request-payload tokens exactly one time. + // The single forward-promise finally owner calls it after the forward settles. + // The seen-id token stays charged for the channel lifetime, so it is not part + // of this release; the terminal teardown releases it. + let forwardTokensReleased = false; + const releaseForwardTokens = (): void => { + if (forwardTokensReleased) return; + forwardTokensReleased = true; + releaseToken(requestFrameToken); + releaseToken(requestPayloadToken); + }; const record: DuplexBrokerRequestRecord = { id: frame.id, @@ -770,23 +970,53 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr forwardTimer, responseTimer, dispatchStartMs: record.dispatchStartMs, + forwardSettled: false, + releaseForwardTokens, }); - forwardRequest(frame, { signal: controller.signal }).then( - (result) => { - // Keep the outcome classification consistent with the file path. A - // possibly-committed mutation carries the indeterminate marker header, so - // map it to the indeterminate outcome. Any other result is completed. - const outcome: DuplexResponseOutcome = - result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate" - ? "indeterminate" - : "completed"; - // The host delivered a real response, so the request span outcome is `ok`. - // A host application status (200, a 4xx, a 5xx) is still a delivered - // response; only a broker-synthesized failure below is `error`. - respond(frame.id, result, outcome, "ok"); - }, - (error) => { + // The forward promise has one cleanup owner. The two settle handlers mark the + // forward settled and answer the gateway. The `finally` then releases the + // request tokens exactly one time and removes the request from both the pending + // map and the orphan map. The forward resolves only after the forward and its + // response-body reader both settle, so this owner waits for both. + const markForwardSettled = (): void => { + // Mark the pending entry settled before `respond` runs, so `respond` never + // moves a just-settled forward to the orphan registry. An already-orphaned + // forward needs no flag; its finally owner releases and drains it. + const entry = pending.get(frame.id); + if (entry) entry.forwardSettled = true; + }; + forwardRequest(frame, { signal: controller.signal }) + .then( + (result) => { + markForwardSettled(); + // Keep the outcome classification consistent with the file path. A + // possibly-committed mutation carries the indeterminate marker header, so + // map it to the indeterminate outcome. Any other result is completed. + const outcome: DuplexResponseOutcome = + result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate" + ? "indeterminate" + : "completed"; + // The host delivered a real response, so the request span outcome is `ok`. + // A host application status (200, a 4xx, a 5xx) is still a delivered + // response; only a broker-synthesized failure below is `error`. + respond(frame.id, result, outcome, "ok"); + }, + (error) => { + markForwardSettled(); + return handleForwardRejection(error); + }, + ) + .finally(() => { + pending.delete(frame.id); + orphanedForwards.delete(frame.id); + releaseForwardTokens(); + }); + + // The forward rejection handler. It classifies the rejection by method safety + // and answers the gateway. The `markForwardSettled` call above runs first, so + // the response never re-orphans a settled forward. + function handleForwardRejection(error: unknown): void { if (controller.signal.aborted) { // The forward budget aborted the call. A safe method never changes // host state, so a forward timeout stays retryable for it. Return a @@ -873,8 +1103,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr "indeterminate", "error", ); - }, - ); + } }; const handleFrame = (frame: DuplexFrame): void => { @@ -947,6 +1176,8 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr setState("closing"); clearHeartbeat(); clearPending(); + releaseSeenRequestIdTokens(); + decoder.dispose(); // Send an orderly close frame. Ignore a write failure here; the broker is // already closing, so a dead channel needs no loss record. try { diff --git a/packages/adapter-utils/src/duplex-frame-codec.test.ts b/packages/adapter-utils/src/duplex-frame-codec.test.ts index 42ec35a4dbc..c19008fb558 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.test.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.test.ts @@ -12,6 +12,7 @@ import { type DuplexDecodeResult, type DuplexFrame, } from "./duplex-frame-codec.js"; +import { DuplexAggregateByteLedger } from "./duplex-aggregate-byte-ledger.js"; // One expected decode result in the fixture: either a decoded frame or the code // of a protocol error. The codec must never throw on the read path. @@ -322,6 +323,117 @@ describe("request id byte bound", () => { }); }); +describe("decoder aggregate byte ledger charging", () => { + // The host injects the process-owned ledger into the decoder. The decoder + // charges the raw partial-frame bytes it retains between chunks, plus the peak + // replacement buffer it allocates on concat. It never retains an uncharged + // buffer, and it fails closed when a reservation would pass the ceiling. + function makeLedger(ceilingBytes = 1024): DuplexAggregateByteLedger { + return new DuplexAggregateByteLedger({ ceilingBytes }); + } + + it("charges the retained partial frame, then releases it when the frame completes", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + const line = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const bytes = Buffer.from(line, "utf8"); + + // A partial frame with no newline stays retained and stays charged. + const first = decoder.push(bytes.subarray(0, 3)); + expect(first).toHaveLength(0); + expect(ledger.bytesInUse).toBe(3); + expect(ledger.liveTokenCount).toBe(1); + + // The completing chunk delivers the frame and releases the retained bytes. + const second = decoder.push(bytes.subarray(3)); + expect(second).toHaveLength(1); + expect(second[0].ok).toBe(true); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("covers the peak concat buffer and settles on the remaining bytes", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + const first = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const partial = '{"version":1,"type":"hea'; + // One full frame plus a partial second frame arrive in one chunk. The full + // frame decodes and releases; the partial tail stays charged at its exact size. + const results = decoder.push(Buffer.from(first + partial, "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(true); + expect(ledger.bytesInUse).toBe(Buffer.byteLength(partial, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + }); + + it("fails closed when a chunk would pass the ceiling and retains nothing", () => { + const ledger = makeLedger(16); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + const results = decoder.push(Buffer.from("z".repeat(64), "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(false); + if (!results[0].ok) { + expect(results[0].error.code).toBe("aggregate_bytes_exceeded"); + } + // The decoder retained nothing, so the ledger stays at zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("resynchronizes on the next newline after an aggregate rejection", () => { + const ledger = makeLedger(64); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + // A first chunk over the ceiling fails closed. + const rejected = decoder.push(Buffer.from("z".repeat(128), "utf8")); + expect(rejected[0].ok).toBe(false); + + // The next chunk starts with a newline, so the decoder drops the stale tail + // and decodes the good frame that follows. + const good = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const results = decoder.push(Buffer.from(`\n${good}`, "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(true); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("releases the retained token when an oversized frame is discarded", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ maxFrameBytes: 16, aggregateByteLedger: ledger }); + // An oversized frame with no newline is discarded and its bytes are released. + const results = decoder.push(Buffer.from("z".repeat(64), "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(false); + if (!results[0].ok) expect(results[0].error.code).toBe("frame_too_large"); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("releases the retained partial frame on dispose", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + decoder.push(Buffer.from('{"version":1', "utf8")); + expect(ledger.bytesInUse).toBeGreaterThan(0); + decoder.dispose(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + // A second dispose is a no-op and records no accounting defect. + decoder.dispose(); + expect(ledger.bytesInUse).toBe(0); + }); + + it("charges nothing when the caller injects no ledger", () => { + const decoder = new DuplexFrameDecoder(); + const results = decoder.push(Buffer.from('{"version":1', "utf8")); + expect(results).toHaveLength(0); + // No ledger means no charge; the decoder still buffers and decodes as before. + const line = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const done = decoder.push(Buffer.from(`}\n${line}`, "utf8")); + // The stitched first frame is malformed JSON; the second is a valid heartbeat. + expect(done.some((result) => result.ok)).toBe(true); + }); +}); + describe("size-checked encode", () => { it("runs every shared encode vector and matches the expected result", () => { // Every codec copy runs the same encode vectors. This copy proves it enforces diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index 20cd814506a..c1723ad763c 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -17,6 +17,12 @@ * keeps one bad frame from crashing the read loop. */ +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + type DuplexAggregateByteLedger, + type ReservationToken, +} from "./duplex-aggregate-byte-ledger.js"; + /** The wire version this codec reads and writes. */ export const DUPLEX_FRAME_VERSION = 1; @@ -134,7 +140,8 @@ export type DuplexProtocolErrorCode = | "unknown_type" | "version_mismatch" | "frame_too_large" - | "id_too_large"; + | "id_too_large" + | "aggregate_bytes_exceeded"; /** A decode-time protocol error. The read path returns it; it never throws. */ export interface DuplexProtocolError { @@ -344,6 +351,18 @@ function validateError(frame: Record): DuplexDecodeResult { export interface DuplexFrameDecoderOptions { /** The maximum size of one frame, in bytes. Defaults to {@link DEFAULT_MAX_DUPLEX_FRAME_BYTES}. */ maxFrameBytes?: number; + /** + * The process-owned aggregate byte ledger. When present, the decoder reserves + * the exact bytes of the raw partial frame it retains between chunks, and the + * peak replacement buffer it allocates on concat, against this ledger under the + * `decoder_buffer` owner. The host injects the same ledger object it injects at + * every other retention site, so one gauge bounds the aggregate retained bytes. + * When absent the decoder charges nothing and behaves as before. + * + * The generated sandbox decoder runs in a separate operating-system process. It + * cannot share this host ledger. It receives its own separate cap instead. + */ + aggregateByteLedger?: DuplexAggregateByteLedger; } /** @@ -364,15 +383,41 @@ export class DuplexFrameDecoder { private buffer: Buffer = EMPTY; private discarding = false; private readonly maxFrameBytes: number; + private readonly ledger: DuplexAggregateByteLedger | null; + /** + * The token for the bytes currently retained in `this.buffer`. Its byte amount + * equals `this.buffer.length` at the end of every `push`. It is `null` when the + * buffer is empty or the decoder has no ledger. + */ + private retainedToken: ReservationToken | null = null; constructor(options: DuplexFrameDecoderOptions = {}) { this.maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES; + this.ledger = options.aggregateByteLedger ?? null; } /** Feed one chunk. Return the frames and protocol errors that complete on it. */ push(chunk: Buffer | Uint8Array | string): DuplexDecodeResult[] { const incoming = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk); + + // Reserve the incoming bytes before the concat allocates the replacement + // buffer. The retained token already covers the current buffer, so the two + // tokens together cover the peak `old + incoming` allocation. A rejected + // reservation fails closed: the decoder drops the incoming chunk, releases + // the retained buffer, resynchronizes at the next newline, and reports the + // fixed aggregate rejection marker. It retains nothing uncharged. + let incomingToken: ReservationToken | null = null; + if (this.ledger && incoming.length > 0) { + incomingToken = this.ledger.reserve("decoder_buffer", incoming.length); + if (incomingToken === null) { + this.releaseRetained(); + this.buffer = EMPTY; + this.discarding = true; + return [fail("aggregate_bytes_exceeded", DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED)]; + } + } + this.buffer = this.buffer.length === 0 ? incoming : Buffer.concat([this.buffer, incoming]); @@ -411,6 +456,54 @@ export class DuplexFrameDecoder { } results.push(decodeDuplexLine(line)); } + + // Reconcile the ledger to the bytes still retained in `this.buffer`. Release + // the old retained token and the incoming token, then reserve one token for + // the remainder. The remainder is a subset of the just-released + // `old + incoming` bytes, so this reserve always fits. + this.reconcileRetained(incomingToken); return results; } + + /** + * Release the retained-buffer token and drop the buffer. The host calls this at + * channel teardown, so the decoder never leaks a `decoder_buffer` token after + * the channel ends. A second call is a no-op, because the field is already + * `null`. + */ + dispose(): void { + this.releaseRetained(); + this.buffer = EMPTY; + this.discarding = false; + } + + /** Release the retained-buffer token one time and clear the field. */ + private releaseRetained(): void { + if (this.ledger && this.retainedToken) { + this.ledger.release(this.retainedToken); + } + this.retainedToken = null; + } + + /** + * Move the ledger charge to the bytes still in `this.buffer`. Release the old + * retained token and the `incoming` token, then reserve one token for the + * remaining bytes. A `null` reserve here is an accounting defect, because the + * remainder never passes the just-released bytes; the decoder fails closed and + * drops the buffer so it retains nothing uncharged. + */ + private reconcileRetained(incomingToken: ReservationToken | null): void { + if (!this.ledger) return; + this.releaseRetained(); + if (incomingToken) { + this.ledger.release(incomingToken); + } + if (this.buffer.length > 0) { + this.retainedToken = this.ledger.reserve("decoder_buffer", this.buffer.length); + if (this.retainedToken === null) { + this.buffer = EMPTY; + this.discarding = true; + } + } + } } diff --git a/packages/adapter-utils/src/duplex-telemetry.ts b/packages/adapter-utils/src/duplex-telemetry.ts index 2bcf8073cbd..b31c6559183 100644 --- a/packages/adapter-utils/src/duplex-telemetry.ts +++ b/packages/adapter-utils/src/duplex-telemetry.ts @@ -35,6 +35,40 @@ export const DUPLEX_COUNTER_LOSS_TOTAL = "sandbox_duplex_loss_total"; /** The guarded counter for one leaked provider session on teardown. */ export const DUPLEX_COUNTER_SESSION_LEAK_TOTAL = "sandbox_duplex_session_leak_total"; +/** + * The process-scoped gauge for the aggregate retained bytes across every live + * duplex route. The host aggregate byte ledger sets it on each reserve and each + * release. The record carries no dynamic dimension. + */ +export const DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE = "sandbox_duplex_aggregate_bytes_in_use"; +/** + * The counter for one rejected aggregate byte reservation. The host ledger + * increments it when a reservation would pass the aggregate ceiling. The record + * carries no dynamic dimension. + */ +export const DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL = + "sandbox_duplex_aggregate_byte_reservation_rejections_total"; +/** + * The counter for one aggregate byte accounting defect. The host ledger + * increments it on a double release or a transfer of a token it does not hold. + * The record carries no dynamic dimension. + */ +export const DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL = + "sandbox_duplex_aggregate_byte_accounting_underflow_total"; + +/** + * The closed set of aggregate byte ledger metric names. A test pins this exact + * set, so a new ledger metric name needs an explicit review. Each record uses + * only closed constant dimensions and no dynamic label. The telemetry contract + * documents these metrics under "Aggregate byte ledger metrics" in + * `packages/shared/src/telemetry/README.md`. + */ +export const DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES = [ + DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, + DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, + DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, +] as const; + /** * The closed dimension-key set. Every span attribute, counter label, and event * field uses only these keys. A test asserts the exact set, so a new key never @@ -64,6 +98,9 @@ export type DuplexOutcomeValue = "ok" | "error"; * stage: the process-scoped route ceiling was full (`route_busy`), the entrypoint * sync failed (`entrypoint_sync_failed`), the broker construction failed * (`broker_construction_failed`), or the channel open failed (`channel_open_failed`). + * The `aggregate_bytes_exceeded` reason names a readiness handshake the host fell + * back because the process aggregate byte ceiling had no room for the readiness + * buffer. */ export type DuplexFallbackReason = | "gate_off" @@ -75,7 +112,8 @@ export type DuplexFallbackReason = | "ready_invalid" | "ready_nonce_mismatch" | "ready_timeout" - | "contaminated"; + | "contaminated" + | "aggregate_bytes_exceeded"; /** The class of a terminal loss, relative to the first request dispatch. */ export type DuplexLossClass = "pre_dispatch" | "post_dispatch"; diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 73660142f9f..0ecf167102c 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -43,12 +43,18 @@ import { type StartupTraceContext, type StartupTracer, } from "./acpx-engine/startup-timing.js"; +import { + DuplexAggregateByteLedger, + DUPLEX_AGGREGATE_TOKEN_OWNERS, + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, +} from "./duplex-aggregate-byte-ledger.js"; import { createSandboxRunLogTailFactory, type SandboxRunLogTailFactory } from "./sandbox-run-log-stream.js"; import { runChildProcess } from "./server-utils.js"; import { shellQuote } from "./ssh.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; import { DEFAULT_MAX_DUPLEX_FRAME_BYTES, + DuplexFrameDecoder, DUPLEX_FRAME_VERSION, decodeDuplexLine, encodeDuplexFrame, @@ -66,10 +72,14 @@ import { } from "./duplex-bridge-broker.js"; import { createDuplexTelemetry, + DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES, + DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, + DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, DUPLEX_COUNTER_CHANNEL_OPEN_TOTAL, DUPLEX_COUNTER_FALLBACK_TOTAL, DUPLEX_COUNTER_LOSS_TOTAL, DUPLEX_DIMENSION_KEYS, + DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, DUPLEX_SPAN_CHANNEL_OPEN, DUPLEX_SPAN_REQUEST, DUPLEX_TRANSPORT_EVENT, @@ -2678,6 +2688,155 @@ describe("sandbox adapter execution targets", () => { } }); + it("charges the response-body bytes against the host aggregate ledger and releases every token on success", async () => { + // The host stamps one process-owned aggregate byte ledger on the sandbox + // target. The forward response-body reader charges its retained bytes against + // that ledger. A successful read charges the chunk bytes and the + // concatenation buffer, then releases every token, so the ledger returns to + // zero after the forward completes. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-ok-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); + await mkdir(runtimeRootDir, { recursive: true }); + + const responseBody = JSON.stringify({ id: "issue-1" }); + const apiServer = createServer((_req, res) => { + res.writeHead(200, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(responseBody, "utf8")), + }); + res.end(responseBody); + }); + await new Promise((resolve, reject) => { + apiServer.once("error", reject); + apiServer.listen(0, "127.0.0.1", () => resolve()); + }); + const address = apiServer.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the bridge test API server to listen on a TCP port."); + } + + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd, + runner: createLocalSandboxRunner(), + timeoutMs: 30_000, + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-bridge-ledger-ok", + target, + runtimeRootDir, + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: `http://127.0.0.1:${address.port}`, + maxBodyBytes: 512, + }); + try { + const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, { + method: "GET", + headers: { + authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, + }, + }); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ id: "issue-1" }); + // The reader released every token, so the aggregate gauge and the live-token + // registry both return to zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + await bridge?.stop(); + await new Promise((resolve) => apiServer.close(() => resolve())); + } + }); + + it("fails a response-body read closed when the host aggregate ledger has no room and retains no bytes", async () => { + // The aggregate ledger sits at a tiny ceiling, so a response body larger than + // the ceiling cannot reserve its bytes. The reader fails closed: it cancels + // the stream reader, retains nothing, and reports the fixed marker. The safe + // GET maps the marker to a retryable 502. The ledger returns to zero, because + // the reader released the tokens it held before the rejection. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-full-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); + await mkdir(runtimeRootDir, { recursive: true }); + + // The body sits under the per-request size limit but over the aggregate + // ceiling, so the aggregate ledger, not the per-request limit, rejects it. + const responseBody = "x".repeat(256); + const apiServer = createServer((_req, res) => { + res.writeHead(200, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(responseBody, "utf8")), + }); + res.end(responseBody); + }); + await new Promise((resolve, reject) => { + apiServer.once("error", reject); + apiServer.listen(0, "127.0.0.1", () => resolve()); + }); + const address = apiServer.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the bridge test API server to listen on a TCP port."); + } + + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd, + runner: createLocalSandboxRunner(), + timeoutMs: 30_000, + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-bridge-ledger-full", + target, + runtimeRootDir, + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: `http://127.0.0.1:${address.port}`, + maxBodyBytes: 4096, + }); + try { + const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, { + method: "GET", + headers: { + authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, + }, + }); + + // The safe GET maps the aggregate rejection to a retryable 502 with no + // indeterminate marker. The body carries only the fixed rejection marker. + expect(response.status).toBe(502); + expect(response.headers.get("x-paperclip-bridge-outcome")).toBeNull(); + await expect(response.json()).resolves.toEqual({ + error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + }); + // The reader released the tokens it held before the rejection, so the + // aggregate gauge and the live-token registry both return to zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + await bridge?.stop(); + await new Promise((resolve) => apiServer.close(() => resolve())); + } + }); + it("forwards the host indeterminate-outcome header so the sandbox server maps the 504 to a non-retryable 409", async () => { // The host marks a possibly-committed mutation with a 504 and the // `x-paperclip-bridge-outcome: indeterminate` header. The forward must keep @@ -3130,6 +3289,66 @@ describe("sandbox adapter execution targets", () => { expect(control.stopCount).toBeGreaterThanOrEqual(1); }, 20000); + it("falls back to the file bridge when a post-READY pre-bind flood exceeds the aggregate ceiling", async () => { + // The gateway sends a valid READY, then floods the channel before the broker + // binds. The pre-READY buffer cap does not bound the post-READY replay buffer, + // so the replay reservation must. The ceiling admits the small READY frame but + // rejects the flood. The host drops the buffer, stops the channel, and selects + // the file bridge with the aggregate marker. No request forwards, and the + // aggregate ledger returns to zero. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-replay-flood-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + await mkdir(remoteCwd, { recursive: true }); + const api = await startRecordingApiServer(); + // The flood is larger than the ceiling; the READY frame is far smaller. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4096 }); + const { runner, control } = makeDuplexSelectionRunner((ctx) => { + ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitRaw("x".repeat(64 * 1024)); + }); + const { recorder, counters } = createRecordingDuplexRecorder(); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "daytona", + remoteCwd, + timeoutMs: 30_000, + runner, + effectiveCapabilities: duplexCapabilities(true), + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-duplex-replay-flood", + target, + runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: api.origin, + enableSandboxDuplexBridge: true, + duplexTelemetryRecorder: recorder, + }); + try { + // The file bridge serves, not the duplex transport. + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1"); + // The fallback names the aggregate marker on the file transport. + const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL); + expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded"); + expect(fallback?.dimensions.transport).toBe("file"); + // The gate stopped the flooded channel. + expect(control.stopCount).toBeGreaterThanOrEqual(1); + // No request forwarded, because the broker never bound. + expect(api.requests).toHaveLength(0); + // The aggregate ledger returns to zero with no live token. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + await bridge?.stop(); + await api.close(); + } + }, 20000); + it("streams run logs on the duplex path under the same gate and log line as the file path", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-runlog-")); cleanupDirs.push(rootDir); @@ -3583,6 +3802,24 @@ describe("sandbox adapter execution targets", () => { ]); }); + it("pins the exact aggregate byte ledger metric names", () => { + // The aggregate byte ledger metric names are closed. This test locks the + // exact set, so a new gauge or counter name needs an explicit change here. + // Each record carries only closed constant dimensions and no dynamic label. + expect([...DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES]).toEqual([ + "sandbox_duplex_aggregate_bytes_in_use", + "sandbox_duplex_aggregate_byte_reservation_rejections_total", + "sandbox_duplex_aggregate_byte_accounting_underflow_total", + ]); + expect(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE).toBe("sandbox_duplex_aggregate_bytes_in_use"); + expect(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL).toBe( + "sandbox_duplex_aggregate_byte_reservation_rejections_total", + ); + expect(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL).toBe( + "sandbox_duplex_aggregate_byte_accounting_underflow_total", + ); + }); + it("records a duplex request span with latency and the fixed dimension keys", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-obs-")); cleanupDirs.push(rootDir); @@ -3701,6 +3938,7 @@ describe("sandbox adapter execution targets", () => { "ready_nonce_mismatch", "ready_timeout", "contaminated", + "aggregate_bytes_exceeded", ]; expect(approvedReasons).toContain(fallback?.dimensions.fallback_reason); expect(fallback?.dimensions).toMatchObject({ transport: "file", outcome: "error" }); @@ -4213,6 +4451,128 @@ describe("sandbox adapter execution targets", () => { } }, 20000); + it("fails the readiness handshake closed when the host aggregate ledger has no room for the pre-READY buffer", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-ready-ledger-full-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + await mkdir(remoteCwd, { recursive: true }); + const api = await startRecordingApiServer(); + // The host stamps one process-owned aggregate byte ledger on the sandbox + // target at a tiny ceiling. The fake gateway sends a pre-READY blob larger + // than the ceiling, so the gate cannot reserve the blob bytes. The gate fails + // closed: it retains nothing, records the aggregate fallback reason, and falls + // back to the file bridge. The blob is smaller than the readiness buffer cap, + // so the aggregate ledger, not the buffer cap, drives the failure. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 256 }); + const preReadyBlob = "x".repeat(4_096); + const { runner, control } = makeDuplexSelectionRunner((ctx) => { + ctx.emitRaw(preReadyBlob); + }); + const { recorder, counters } = createRecordingDuplexRecorder(); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "daytona", + remoteCwd, + timeoutMs: 30_000, + runner, + effectiveCapabilities: duplexCapabilities(true), + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-ready-ledger-full", + target, + runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: api.origin, + enableSandboxDuplexBridge: true, + // A long readiness timeout, so the aggregate ledger, not the timeout, drives + // the failure. + duplexReadinessTimeoutMs: 5_000, + duplexTelemetryRecorder: recorder, + }); + try { + expect(bridge).not.toBeNull(); + expect(control.openCount).toBe(1); + // The aggregate rejection drove the failure, so the file bridge serves. + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1"); + const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL); + expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded"); + // The gate retained nothing after the rejection, so the aggregate gauge and + // the live-token registry both return to zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(control.closeCount + control.stopCount).toBeGreaterThanOrEqual(1); + } finally { + await bridge?.stop(); + await api.close(); + } + }, 20000); + + it("charges the pre-READY buffer against the injected host ledger and releases it when readiness passes", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-ready-ledger-ok-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + await mkdir(remoteCwd, { recursive: true }); + const api = await startRecordingApiServer(); + // The host stamps one process-owned aggregate byte ledger on the sandbox + // target at a generous ceiling. The fake gateway sends one pre-READY noise + // line, then the valid READY frame. The gate charges the noise bytes against + // the injected ledger, passes readiness, and releases the pre-READY tokens. + // The broker's frame decoder re-charges any post-READY bytes, so the ledger + // returns to zero after the handshake. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 }); + const reserveSpy = vi.spyOn(ledger, "reserve"); + const { runner, control } = makeDuplexSelectionRunner((ctx) => { + ctx.emitRaw("pty-echo-noise"); + ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "daytona", + remoteCwd, + timeoutMs: 30_000, + runner, + effectiveCapabilities: duplexCapabilities(true), + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-ready-ledger-ok", + target, + runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: api.origin, + enableSandboxDuplexBridge: true, + duplexReadinessTimeoutMs: 5_000, + }); + try { + expect(bridge).not.toBeNull(); + // Readiness passed, so the duplex transport serves. + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); + // The gate charged the pre-READY noise against the exact injected ledger, so + // the identity holds at this seam. + expect(reserveSpy).toHaveBeenCalledWith("readiness_buffer", expect.any(Number)); + // The gate released every readiness-buffer token on settle, so the aggregate + // gauge and the live-token registry both return to zero. + await waitForCondition( + () => ledger.bytesInUse === 0 && ledger.liveTokenCount === 0, + "the readiness gate to release every pre-READY token", + 4000, + ); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + reserveSpy.mockRestore(); + await bridge?.stop(); + await api.close(); + } + }, 20000); + it("bounds the pre-READY newline-scan work by the bytes received", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-scan-")); cleanupDirs.push(rootDir); @@ -4850,11 +5210,16 @@ interface EmbeddedCodec { encodeDuplexFrame: (frame: unknown) => string; encodeDuplexFrameChecked: (frame: unknown, maxFrameBytes?: number) => EmbeddedEncodeResult; decodeDuplexLine: (line: string | Buffer) => EmbeddedDecodeResult; - DuplexFrameDecoder: new (options?: { maxFrameBytes?: number }) => { + DuplexFrameDecoder: new (options?: { maxFrameBytes?: number; maxAggregateBytes?: number }) => { push: (chunk: Buffer) => EmbeddedDecodeResult[]; + scope: string; + bytesInUse: number; + aggregateRejections: number; }; DUPLEX_FRAME_VERSION: number; DEFAULT_MAX_DUPLEX_FRAME_BYTES: number; + DUPLEX_DECODER_SCOPE: string; + DEFAULT_MAX_DUPLEX_DECODER_BYTES: number; } type ExpectedVectorResult = { frame: unknown } | { error: string }; @@ -5099,7 +5464,7 @@ describe("sandbox duplex gateway", () => { it("embedded gateway codec passes every vector in the shared fixture", async () => { const codecFactory = new Function( - `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES };`, + `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`, ) as unknown as () => EmbeddedCodec; const codec = codecFactory(); @@ -5199,6 +5564,61 @@ describe("sandbox duplex gateway", () => { expect(encodeFailures).toEqual([]); }); + it("bounds the sandbox decoder with a separate sandbox_process scope, distinct from the host ledger scope", () => { + // The generated gateway runs in a separate operating-system process, so its + // decoder cannot share the host aggregate byte ledger. It enforces a separate + // local cap under the `sandbox_process` scope. This test wraps the embedded + // source and the host decoder, then proves the two scopes never overlap and the + // sandbox cap fails closed. + const codecFactory = new Function( + `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`, + ) as unknown as () => EmbeddedCodec; + const codec = codecFactory(); + + // The sandbox scope is the fixed `sandbox_process` label. + expect(codec.DUPLEX_DECODER_SCOPE).toBe("sandbox_process"); + expect(codec.DEFAULT_MAX_DUPLEX_DECODER_BYTES).toBeGreaterThan(codec.DEFAULT_MAX_DUPLEX_FRAME_BYTES); + // The two scopes are distinct: the host owner set never carries the sandbox + // scope, so the sandbox counter can never map to a host aggregate token. + expect((DUPLEX_AGGREGATE_TOKEN_OWNERS as readonly string[]).includes("sandbox_process")).toBe(false); + + // The sandbox decoder tracks its own `sandbox_process` counter. A frame under + // the cap charges the local counter, and the counter returns to zero once the + // frame drains. + const sandboxDecoder = new codec.DuplexFrameDecoder({ maxAggregateBytes: 64 }); + expect(sandboxDecoder.scope).toBe("sandbox_process"); + const partial = sandboxDecoder.push(Buffer.from('{"version":1,', "utf8")); + expect(partial).toEqual([]); + expect(sandboxDecoder.bytesInUse).toBe(Buffer.byteLength('{"version":1,', "utf8")); + sandboxDecoder.push(Buffer.from('"type":"heartbeat"}\n', "utf8")); + expect(sandboxDecoder.bytesInUse).toBe(0); + + // A chunk over the local cap fails closed. The decoder retains nothing, reports + // the aggregate rejection, and increments only its local `sandbox_process` + // counter. + const cappedDecoder = new codec.DuplexFrameDecoder({ maxAggregateBytes: 8 }); + const overCap = cappedDecoder.push(Buffer.from("x".repeat(64), "utf8")); + expect(overCap.length).toBe(1); + const rejection = overCap[0]; + expect(rejection.ok).toBe(false); + expect(rejection.error?.code).toBe("aggregate_bytes_exceeded"); + expect(cappedDecoder.bytesInUse).toBe(0); + expect(cappedDecoder.aggregateRejections).toBe(1); + + // The host decoder charges the host aggregate byte ledger under the + // `decoder_buffer` owner. It never uses the sandbox scope. This proves the two + // implementations use distinct scopes: the host uses the injected ledger; the + // sandbox uses its local counter. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 }); + const hostDecoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + hostDecoder.push(Buffer.from('{"version":1,', "utf8")); + expect(ledger.bytesInUse).toBe(Buffer.byteLength('{"version":1,', "utf8")); + expect(ledger.liveTokenCount).toBeGreaterThan(0); + hostDecoder.dispose(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + it("returns the same HTTP response as the file gateway for a forwarded request", async () => { const token = "duplex-token-forward"; const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token }); @@ -6172,3 +6592,213 @@ describe("CLI-lane run-disposition seam", () => { expect(settleCalls).toBe(0); }); }); + +describe("duplex readiness gate replay-buffer reservation", () => { + const READY_NONCE = "0123456789abcdef0123456789abcdef"; + + // A fake duplex channel the test drives directly. `control.emitData` re-enters + // the data listener the gate bound at construction. `control.emitExit` re-enters + // the exit listener. The fake records the stop and the close calls. + function makeFakeReadinessChannel(): { + channel: CommandManagedDuplexChannel; + control: { + stopCount: number; + closeCount: number; + written: string[]; + emitData: (chunk: string) => void; + emitExit: (exit: { exitCode: number | null }) => void; + }; + } { + let dataListener: ((chunk: string) => void) | null = null; + let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; + const control = { + stopCount: 0, + closeCount: 0, + written: [] as string[], + emitData: (chunk: string): void => dataListener?.(chunk), + emitExit: (exit: { exitCode: number | null }): void => exitListener?.(exit), + }; + const channel: CommandManagedDuplexChannel = { + write(data: string): void { + control.written.push(data); + }, + onData(listener: (chunk: string) => void): void { + dataListener = listener; + }, + onExit(listener: (exit: { exitCode: number | null }) => void): void { + exitListener = listener; + }, + stop(): void { + control.stopCount += 1; + }, + close(): Promise { + control.closeCount += 1; + return Promise.resolve(); + }, + }; + return { channel, control }; + } + + // A ledger that counts the reservation-rejection and the accounting-underflow + // signals, so a test proves the one-owner-one-release invariant holds. + function makeCountingLedger(ceilingBytes: number): { + ledger: DuplexAggregateByteLedger; + counts: { rejections: number; underflows: number }; + } { + const counts = { rejections: 0, underflows: 0 }; + const ledger = new DuplexAggregateByteLedger({ + ceilingBytes, + telemetry: { + setBytesInUse(): void {}, + recordReservationRejection(): void { + counts.rejections += 1; + }, + recordAccountingUnderflow(): void { + counts.underflows += 1; + }, + }, + }); + return { ledger, counts }; + } + + function readyLine(): string { + return `${JSON.stringify({ version: 1, type: "ready", nonce: READY_NONCE })}\n`; + } + + it("charges the post-READY suffix and releases it after the broker handoff", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + const suffix = "hello-post-ready-suffix"; + // The READY line and the suffix arrive in one chunk. The gate drops the whole + // pre-READY buffer charge, then charges only the retained suffix. + control.emitData(`${readyLine()}${suffix}`); + const readiness = await gate.ready; + expect(readiness.ok).toBe(true); + expect(gate.replayOverflowed()).toBe(false); + // The gate holds the suffix under one readiness_replay token before the bind. + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + // The broker binds and replays the suffix; the gate releases the token after + // the synchronous handoff. + const replayed: string[] = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + expect(replayed).toEqual([suffix]); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(counts.underflows).toBe(0); + }); + + it("releases the suffix token on disposePendingReplay without a broker handoff", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + const suffix = "abandoned-suffix"; + control.emitData(`${readyLine()}${suffix}`); + expect((await gate.ready).ok).toBe(true); + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + // A broker-construction failure abandons the buffer, so the caller disposes it. + gate.disposePendingReplay(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + // A second dispose is a no-op and never underflows. + gate.disposePendingReplay(); + expect(ledger.bytesInUse).toBe(0); + expect(counts.underflows).toBe(0); + }); + + it("releases the suffix token after a pre-bind exit then a broker handoff", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + const suffix = "pre-bind-exit-suffix"; + control.emitData(`${readyLine()}${suffix}`); + expect((await gate.ready).ok).toBe(true); + // The channel exits after READY but before the broker binds. The gate holds the + // exit and keeps the pending suffix charged. + control.emitExit({ exitCode: 0 }); + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + // The broker binds, replays the suffix and the exit, then the gate releases the + // reservation. + const replayed: string[] = []; + const exits: Array<{ exitCode: number | null }> = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + gate.brokerChannel.onExit((exit) => exits.push(exit)); + expect(replayed).toEqual([suffix]); + expect(exits).toEqual([{ exitCode: 0 }]); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(counts.underflows).toBe(0); + }); + + it("fails closed when a post-READY pre-bind chunk floods past the ceiling", async () => { + const { channel, control } = makeFakeReadinessChannel(); + // The ceiling admits the small READY line but not the flood chunk. + const { ledger, counts } = makeCountingLedger(256); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + // The READY line arrives alone, so the pending suffix starts empty. + control.emitData(readyLine()); + expect((await gate.ready).ok).toBe(true); + expect(ledger.bytesInUse).toBe(0); + // A post-READY chunk larger than the ceiling floods the replay buffer before + // the broker binds. The gate refuses the reservation and fails closed. + control.emitData("x".repeat(512)); + expect(gate.replayOverflowed()).toBe(true); + expect(control.stopCount).toBe(1); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(counts.rejections).toBe(1); + expect(counts.underflows).toBe(0); + // Binding the broker replays nothing, because the gate dropped the buffer. + const replayed: string[] = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + expect(replayed).toEqual([]); + }); + + it("drops the retained pre-READY buffer on READY acceptance", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + // A large pre-READY noise line and a large suffix arrive with the READY line + // in one chunk. A sandbox controls every byte here. + const noise = `${"n".repeat(4096)}\n`; + const suffix = "s".repeat(2048); + control.emitData(`${noise}${readyLine()}${suffix}`); + expect((await gate.ready).ok).toBe(true); + // The gate drops the pre-READY buffer, so the process no longer retains the + // noise prefix. Without this, the process holds the full sandbox string while + // the ledger charges only the suffix, so retention passes the ceiling. + expect(gate.retainedReadinessBufferLength()).toBe(0); + // The ledger charges only the retained suffix, not the dropped prefix. + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + // The broker binds, replays the suffix, and the gate releases the token. + const replayed: string[] = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + expect(replayed).toEqual([suffix]); + expect(ledger.bytesInUse).toBe(0); + expect(gate.retainedReadinessBufferLength()).toBe(0); + expect(counts.underflows).toBe(0); + }); +}); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index faece1d9bf0..065a7f8c5b7 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -29,6 +29,7 @@ import { createSandboxCallbackBridgeAsset, createSandboxCallbackBridgeToken, DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES, + DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES, SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE, SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT, sandboxCallbackBridgeDirectories, @@ -63,6 +64,11 @@ import { type DuplexFallbackReason, type DuplexTelemetryRecorder, } from "./duplex-telemetry.js"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + type DuplexAggregateByteLedger, + type ReservationToken, +} from "./duplex-aggregate-byte-ledger.js"; import { createSshCommandManagedRuntimeRunner, parseSshRemoteExecutionSpec, runSshCommand, shellQuote } from "./ssh.js"; import { ensureCommandResolvable, @@ -184,6 +190,16 @@ export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWor * observability surface. Absent means the safe no-op default. */ duplexTelemetryRecorder?: DuplexTelemetryRecorder | null; + /** + * The process-owned aggregate byte ledger for the sandbox duplex channel. The + * host stamps this same object on every sandbox target on the same seam as + * `runner`, so one shared gauge bounds the aggregate retained bytes across all + * live duplex routes. The live object stays on the host and never enters the + * sandbox environment. The bridge passes it to the broker, the decoder, and the + * response-body reader. Absent means no host ledger; a non-duplex run keeps the + * bridge inert for this seam. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } export type AdapterExecutionTarget = @@ -428,6 +444,20 @@ export function adapterExecutionTargetDuplexTelemetryRecorder( : null; } +/** + * Read the injected aggregate byte ledger off a target. Only a sandbox target + * with a ledger attached returns it. Every other target returns null, so the + * bridge stays inert for this seam. The reader never makes a fresh ledger, so a + * host duplex run always uses the one process-owned ledger the host stamped. + */ +export function adapterExecutionTargetDuplexAggregateByteLedger( + target: AdapterExecutionTarget | null | undefined, +): DuplexAggregateByteLedger | null { + return target?.kind === "remote" && target.transport === "sandbox" + ? target.duplexAggregateByteLedger ?? null + : null; +} + export function adapterExecutionTargetRemoteCwd( target: AdapterExecutionTarget | null | undefined, localCwd: string, @@ -1468,7 +1498,27 @@ function bridgeResponseBodyLimitError(maxBodyBytes: number): Error { return new Error(`Bridge response body exceeded the configured size limit of ${maxBodyBytes} bytes.`); } -async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: number): Promise { +/** + * Read the forward response body into a string. The reader bounds the body with + * two controls. The per-request `maxBodyBytes` limit rejects a body larger than + * the configured per-request ceiling. The optional host aggregate byte ledger + * bounds the retained bytes across all live routes. + * + * The reader charges the ledger for every retained buffer before it allocates + * that buffer. It reserves the exact chunk bytes before it copies a chunk into a + * retained `Buffer`. It reserves the concatenation buffer before it allocates it. + * A reservation that would pass the aggregate ceiling returns no token; the + * reader retains nothing more, cancels the stream reader, and throws the fixed + * marker {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. The `finally` releases + * every token exactly one time, so the reader charges the retained bytes only + * while the raw buffers live and never leaves a token held after it returns or + * throws. + */ +async function readBridgeForwardResponseBody( + response: Response, + maxBodyBytes: number, + ledger?: DuplexAggregateByteLedger | null, +): Promise { const rawContentLength = response.headers.get("content-length"); if (rawContentLength) { const contentLength = Number.parseInt(rawContentLength, 10); @@ -1483,19 +1533,54 @@ async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: n const reader = response.body.getReader(); const chunks: Buffer[] = []; + // Every response-body reservation token the reader holds. The `finally` block + // releases each token one time, so a return, a size error, an aggregate + // rejection, and a read error all release every token. + const tokens: ReservationToken[] = []; let totalBytes = 0; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (!value) continue; - totalBytes += value.byteLength; - if (totalBytes > maxBodyBytes) { - await reader.cancel().catch(() => undefined); - throw bridgeResponseBodyLimitError(maxBodyBytes); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + const chunkBytes = value.byteLength; + totalBytes += chunkBytes; + if (totalBytes > maxBodyBytes) { + await reader.cancel().catch(() => undefined); + throw bridgeResponseBodyLimitError(maxBodyBytes); + } + // Reserve the exact chunk bytes before the host copies the chunk into a + // retained buffer. A rejection fails closed: cancel the stream reader and + // report the fixed marker; the reader retains nothing more. + if (ledger) { + const token = ledger.reserve("response_body", chunkBytes); + if (!token) { + await reader.cancel().catch(() => undefined); + throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + } + tokens.push(token); + } + chunks.push(Buffer.from(value)); + } + // Reserve the concatenation buffer before the reader allocates it. The + // concatenation buffer is a second copy of the body bytes that lives next to + // the chunk buffers during the concatenation, so it is the peak retained + // allocation. A rejection fails closed with the fixed marker. + if (ledger && totalBytes > 0) { + const concatToken = ledger.reserve("response_body", totalBytes); + if (!concatToken) { + throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + } + tokens.push(concatToken); + } + return Buffer.concat(chunks, totalBytes).toString("utf8"); + } finally { + if (ledger) { + for (const token of tokens) { + ledger.release(token); + } } - chunks.push(Buffer.from(value)); } - return Buffer.concat(chunks, totalBytes).toString("utf8"); } const PROCESS_SESSION_PROXY_SCRIPT = "paperclip-process-session-proxy.mjs"; @@ -2394,7 +2479,8 @@ type DuplexReadinessFailure = | "protocol_contamination" | "nonce_mismatch" | "channel_exit" - | "timeout"; + | "timeout" + | "aggregate_bytes_exceeded"; /** The outcome of the duplex readiness handshake. */ type DuplexReadinessResult = @@ -2416,6 +2502,8 @@ function duplexReadinessFallbackReason(reason: DuplexReadinessFailure): DuplexFa return "ready_timeout"; case "channel_exit": return "ready_invalid"; + case "aggregate_bytes_exceeded": + return "aggregate_bytes_exceeded"; } } @@ -2542,6 +2630,13 @@ export const __duplexReadinessTesting = { resetNewlineScanUnits: (): void => { duplexReadinessNewlineScanUnits = 0; }, + // Build one readiness gate over a supplied channel, so a test can drive the + // readiness-replay reservation lifecycle across every terminal path without the + // whole bridge. Production code never reads this factory. + createReadinessGate: ( + channel: CommandManagedDuplexChannel, + options: { nonce: string; timeoutMs: number; ledger?: DuplexAggregateByteLedger | null }, + ) => createDuplexReadinessGate(channel, options), }; interface DuplexReadinessGate { @@ -2554,14 +2649,82 @@ interface DuplexReadinessGate { * the channel. */ readonly brokerChannel: CommandManagedDuplexChannel; + /** + * Report whether a post-READY pre-bind chunk could not reserve its replay bytes + * against the aggregate ledger. On such a refusal the gate drops the pending + * replay buffer and stops the channel. The caller reads this after `ready` + * resolves `ok`, and before it binds the broker. A `true` result means the caller + * must abandon the broker and select the file bridge with the aggregate marker. + */ + replayOverflowed(): boolean; + /** + * Release every held readiness-replay reservation exactly once and drop the + * pending replay buffer. The caller runs this on a terminal path that abandons + * the pending replay without a broker handoff: a readiness failure, a replay + * overflow, or a broker-construction failure. The normal handoff releases the + * reservation inside `brokerChannel.onData`, so a later call here is a no-op. + */ + disposePendingReplay(): void; + /** + * Test-only. Report the length of the retained pre-READY buffer, in UTF-16 code + * units. A test reads this to prove the gate drops the pre-READY buffer on READY + * acceptance, so the process does not retain the sandbox-controlled prefix. + * Production code does not read this. + */ + retainedReadinessBufferLength(): number; } function createDuplexReadinessGate( channel: CommandManagedDuplexChannel, - options: { nonce: string; timeoutMs: number }, + options: { + nonce: string; + timeoutMs: number; + // The one host-process aggregate byte ledger. The gate charges the untrusted + // pre-READY buffer bytes against it, so a pre-READY flood counts toward the + // aggregate ceiling across all live routes. A gate with no ledger stays inert + // for this seam. The gate holds the same object every other host retention + // site holds, so the aggregate identity holds at this seam. + ledger?: DuplexAggregateByteLedger | null; + }, ): DuplexReadinessGate { + const ledger = options.ledger ?? null; let settled = false; let readyOk = false; + // Every readiness-buffer reservation token the gate holds for the pre-READY + // bytes. The gate releases each token one time when it settles or when it + // accepts READY. On a failed handshake the gate drops the buffer, so the release + // frees the untrusted bytes. On READY the gate discards the whole pre-READY + // buffer, then re-charges only the retained suffix under `readiness_replay`. + const retainedTokens: ReservationToken[] = []; + // Every readiness-replay reservation token the gate holds for the post-READY + // suffix and each later pre-bind chunk. The gate releases each token one time + // after the synchronous handoff to the broker, or on a terminal path that + // abandons the pending replay without a broker handoff. + const replayTokens: ReservationToken[] = []; + // The gate sets this when a post-READY pre-bind chunk cannot reserve its replay + // bytes. On that refusal the gate drops the pending buffer and stops the channel. + // The caller reads it through `replayOverflowed` and selects the file bridge. + let replayOverflow = false; + + // Release every readiness-buffer token exactly once and clear the registry. A + // second call is a no-op, because the array is empty. + function releaseReadinessBufferTokens(): void { + if (!ledger) return; + for (const token of retainedTokens) { + ledger.release(token); + } + retainedTokens.length = 0; + } + + // Release every readiness-replay token exactly once and clear the registry. A + // second call is a no-op, because the array is empty. + function releaseReplayTokens(): void { + if (!ledger) return; + for (const token of replayTokens) { + ledger.release(token); + } + replayTokens.length = 0; + } // The raw bytes the host reads before the READY frame completes. The buffer is // append-only, so the O(1) cap check on `buffer.length` stays valid. let buffer = ""; @@ -2590,6 +2753,11 @@ function createDuplexReadinessGate( settled = true; clearTimeout(timer); if (result.ok) readyOk = true; + // Release every readiness-buffer token exactly once. The gate no longer owns + // the pre-READY bytes: a failed handshake drops the buffer. The READY-accept + // path already released these tokens and charged the retained suffix under + // `readiness_replay`, so this call is a no-op there. + releaseReadinessBufferTokens(); resolveReady(result); } @@ -2599,7 +2767,24 @@ function createDuplexReadinessGate( return; } if (readyOk) { - // READY already passed; hold the bytes until the broker binds. + // READY already passed; hold the bytes until the broker binds. Reserve the + // exact UTF-8 bytes under `readiness_replay` before the append, so the replay + // buffer counts toward the aggregate ceiling. A refusal fails closed: the gate + // drops the pending buffer, releases the replay tokens, stops the channel, and + // sets the overflow flag. The caller reads the flag and selects the file bridge + // with the aggregate marker, because `ready` already resolved before this + // synchronous post-READY chunk arrived. + if (ledger) { + const token = ledger.reserve("readiness_replay", Buffer.byteLength(chunk, "utf8")); + if (!token) { + replayOverflow = true; + pending = ""; + releaseReplayTokens(); + channel.stop(); + return; + } + replayTokens.push(token); + } pending += chunk; return; } @@ -2609,6 +2794,18 @@ function createDuplexReadinessGate( // never grows the buffer after the gate settles. return; } + // Reserve the exact UTF-8 bytes of this chunk against the aggregate ledger + // before the gate retains it. The pre-READY buffer holds untrusted bytes, so + // a flood counts toward the process aggregate ceiling. A rejection fails + // closed: the gate retains nothing more and falls back to the file bridge. + if (ledger) { + const token = ledger.reserve("readiness_buffer", Buffer.byteLength(chunk, "utf8")); + if (!token) { + finish({ ok: false, reason: "aggregate_bytes_exceeded" }); + return; + } + retainedTokens.push(token); + } // Append the new bytes and continue the newline search from `scanFrom`, the // first index not yet examined. Each code unit is read at most one time for // the search, so the total scan work stays linear in the bytes received. @@ -2682,8 +2879,33 @@ function createDuplexReadinessGate( finish({ ok: false, reason: "nonce_mismatch" }); return; } - // Hold the bytes that follow the READY line for the broker to replay. - pending = buffer.slice(newlineIndex + 1); + // The bytes that follow the READY line become the replay buffer for the + // broker. Drop the whole pre-READY buffer charge first, then reserve the + // retained suffix under `readiness_replay`. The release-before-reserve order + // keeps the transient charge equal to the suffix, not the sum of the dropped + // prefix and the retained suffix. The two steps run in one synchronous + // section, so no other route can take the freed bytes in between. + const suffix = buffer.slice(newlineIndex + 1); + // Drop the original pre-READY buffer now. The gate keeps only the + // retained suffix as `pending`, and it charges that suffix under + // `readiness_replay` below. If the gate keeps the buffer, the process + // retains the full sandbox-controlled string while the ledger counts + // only the suffix, so aggregate retention passes the ceiling. This clear + // also covers the broker handoff and the replay disposal. Both run later + // and read no buffer bytes. + buffer = ""; + releaseReadinessBufferTokens(); + if (ledger && suffix.length > 0) { + const token = ledger.reserve("readiness_replay", Buffer.byteLength(suffix, "utf8")); + if (!token) { + // The retained suffix passes the aggregate ceiling. Fail closed: drop + // the suffix and fall back to the file bridge with the aggregate marker. + finish({ ok: false, reason: "aggregate_bytes_exceeded" }); + return; + } + replayTokens.push(token); + } + pending = suffix; finish({ ok: true }); return; } @@ -2723,6 +2945,11 @@ function createDuplexReadinessGate( pending = ""; listener(replay); } + // Release every readiness-replay token exactly once, after the synchronous + // handoff to the broker. The broker charges its own decode retention inside + // the `listener(replay)` call above, so the release here never opens an + // admission gap for the same retained bytes. + releaseReplayTokens(); }, onExit: (listener: (exit: { exitCode: number | null }) => void) => { exitSink = listener; @@ -2736,7 +2963,16 @@ function createDuplexReadinessGate( close: () => channel.close(), }; - return { ready, brokerChannel }; + return { + ready, + brokerChannel, + replayOverflowed: () => replayOverflow, + disposePendingReplay: () => { + pending = ""; + releaseReplayTokens(); + }, + retainedReadinessBufferLength: () => buffer.length, + }; } /** @@ -2817,6 +3053,12 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { } const target = input.target; + // The process-owned aggregate byte ledger the host stamped on this sandbox + // target. The forward response-body reader charges its retained bytes against + // this one ledger, so the aggregate retained bytes across all live routes stay + // under the ceiling. A target with no ledger keeps the reader inert for this + // seam. + const duplexAggregateByteLedger = adapterExecutionTargetDuplexAggregateByteLedger(target); const onLog = input.onLog ?? (async () => {}); const hostApiToken = input.hostApiToken?.trim() ?? ""; if (hostApiToken.length === 0) { @@ -2943,7 +3185,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // to a non-retryable 409 for both the file bridge and the duplex broker. let responseBody: string; try { - responseBody = await readBridgeForwardResponseBody(response, maxBodyBytes); + responseBody = await readBridgeForwardResponseBody( + response, + maxBodyBytes, + duplexAggregateByteLedger, + ); } catch (error) { if (isSafeBridgeMethod(method)) { // The method is safe, so a retry cannot double-apply a mutation. Return a @@ -3049,6 +3295,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { PAPERCLIP_BRIDGE_PORT: String(assignedPort), PAPERCLIP_BRIDGE_NONCE: nonce, PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes), + // The separate sandbox-process raw-decoder cap. The generated gateway runs + // in a different operating-system process, so it cannot share the host + // aggregate byte ledger. It enforces this cap locally under the + // `sandbox_process` scope, and the provider memory allocation bounds it. + PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES: String(DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES), }; const command = buildDuplexGatewayLaunchArgv({ shellCommand, @@ -3080,7 +3331,14 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { } if (channel) { - const gate = createDuplexReadinessGate(channel, { nonce, timeoutMs: readinessTimeoutMs }); + const gate = createDuplexReadinessGate(channel, { + nonce, + timeoutMs: readinessTimeoutMs, + // Inject the one host-process aggregate byte ledger, the same object the + // broker and the response-body reader hold. The gate charges the untrusted + // pre-READY buffer against it, so the aggregate identity holds at this seam. + ledger: duplexAggregateByteLedger, + }); const readiness = await gate.ready; if (!readiness.ok) { // Fail closed. Close the partial channel inside a bounded budget, then @@ -3088,12 +3346,26 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // carries the bridge token reached the channel or any endpoint. The reason // is a fixed enum, so it rides the log line and the fallback telemetry // with no raw value. + gate.disposePendingReplay(); await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); duplexChannelOpen.fallback(duplexReadinessFallbackReason(readiness.reason)); await onLog( "stderr", `[paperclip] Sandbox duplex readiness failed (${readiness.reason}). Using the file bridge.\n`, ); + } else if (gate.replayOverflowed()) { + // Readiness passed, but a post-READY pre-bind chunk passed the aggregate + // byte ceiling. The gate dropped the replay buffer and stopped the channel. + // Release any held replay reservation, close the partial channel inside the + // cleanup budget, and select the file bridge with the aggregate marker. The + // broker never bound, so no request reached the channel or any endpoint. + gate.disposePendingReplay(); + await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); + duplexChannelOpen.fallback(duplexReadinessFallbackReason("aggregate_bytes_exceeded")); + await onLog( + "stderr", + "[paperclip] Sandbox duplex readiness replay exceeded the aggregate byte ceiling (aggregate_bytes_exceeded). Using the file bridge.\n", + ); } else { // Readiness passed. Construct the broker inside the guarded region, so a // construction throw closes the channel within the cleanup budget and @@ -3149,12 +3421,20 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { `[paperclip] Sandbox duplex channel lost (${typedDuplexLossReason(record.reason)}). The run fails.\n`, ); }, + // Inject the one host-process aggregate byte ledger. The broker + // reserves the retained request-frame, request-payload, and no-replay + // set-entry bytes against it, so the aggregate retained bytes across all + // live routes stay under the ceiling. It is the same object the + // response-body reader reads off the stamped target above. + duplexAggregateByteLedger, }); } catch { - // The broker construction failed, so no broker owns the channel. Close - // the channel within the cleanup budget, then select the file bridge. - // The log line names no raw error, so no raw error rides a log line on - // the duplex path. + // The broker construction failed, so no broker owns the channel. The + // broker never bound, so it never released the pending replay reservation; + // release it here. Then close the channel within the cleanup budget and + // select the file bridge. The log line names no raw error, so no raw error + // rides a log line on the duplex path. + gate.disposePendingReplay(); await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); duplexChannelOpen.fallback("broker_construction_failed"); await onLog( diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index 7c81cfde263..9c3d073919f 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -19,6 +19,14 @@ const DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS = 30_000; const DEFAULT_BRIDGE_STOP_TIMEOUT_MS = 2_000; const DEFAULT_BRIDGE_MAX_QUEUE_DEPTH = 64; const DEFAULT_BRIDGE_MAX_BODY_BYTES = 256 * 1024; +// The default cap on the aggregate raw bytes the in-sandbox duplex frame decoder +// retains between chunks. The generated gateway runs in a separate operating-system +// process, so it cannot share the host aggregate byte ledger. It enforces this +// separate cap locally under the `sandbox_process` scope, and the provider memory +// allocation bounds it. The host passes the value through +// PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES. The default is well above one maximum +// frame, so a legitimate single frame never trips it. +const DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES = 8 * 1024 * 1024; // Per-iteration timeout for one poll-loop client call. A healthy control-plane // round trip finishes in well under one second, so 10s is far above a normal // iteration and never false-fires on a slow-but-live call. It is also well @@ -89,6 +97,18 @@ const CALLBACK_BRIDGE_WORKER_FAILED_SPAN = "sandbox.callbackBridge.workerFailed" export const DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES = DEFAULT_BRIDGE_MAX_BODY_BYTES; +/** + * The default cap on the aggregate raw bytes the generated in-sandbox duplex frame + * decoder retains. The host passes it through + * `PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES`. The in-sandbox decoder enforces the + * cap locally under the `sandbox_process` scope; it never touches the host + * aggregate byte ledger. + */ +export const DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES = DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES; + +/** The scope label the in-sandbox duplex decoder reports for its local byte counter. */ +export const SANDBOX_DUPLEX_DECODER_SCOPE = "sandbox_process"; + export interface SandboxCallbackBridgeRouteRule { method: string; path: RegExp; @@ -1746,6 +1766,8 @@ export async function startSandboxCallbackBridgeServer(input: { const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 1; const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 1000000; const DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES = 256; +const DEFAULT_MAX_DUPLEX_DECODER_BYTES = ${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}; +const DUPLEX_DECODER_SCOPE = "${SANDBOX_DUPLEX_DECODER_SCOPE}"; const DUPLEX_NEWLINE_BYTE = 0x0a; const DUPLEX_EMPTY = Buffer.alloc(0); const DUPLEX_RESPONSE_OUTCOMES = new Set(["completed", "indeterminate", "unavailable"]); @@ -1881,10 +1903,32 @@ class DuplexFrameDecoder { this.discarding = false; this.maxFrameBytes = options && options.maxFrameBytes != null ? options.maxFrameBytes : DEFAULT_MAX_DUPLEX_FRAME_BYTES; + // The in-sandbox decoder runs in a separate operating-system process, so it + // cannot share the host aggregate byte ledger. It bounds its own retained + // bytes with a local cap under the "sandbox_process" scope. The counters here + // never increment or release a host aggregate token. + this.scope = DUPLEX_DECODER_SCOPE; + this.maxAggregateBytes = + options && options.maxAggregateBytes != null + ? options.maxAggregateBytes + : DEFAULT_MAX_DUPLEX_DECODER_BYTES; + this.bytesInUse = 0; + this.aggregateRejections = 0; } push(chunk) { const incoming = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk); + // Enforce the local sandbox-process cap before the concat allocates the peak + // "old + incoming" buffer. A rejection fails closed: the decoder drops the + // retained buffer and the incoming chunk, resynchronizes at the next newline, + // and reports the aggregate rejection. It retains nothing over the cap. + if (this.buffer.length + incoming.length > this.maxAggregateBytes) { + this.buffer = DUPLEX_EMPTY; + this.discarding = true; + this.bytesInUse = 0; + this.aggregateRejections += 1; + return [duplexFail("aggregate_bytes_exceeded", "aggregate retained bytes exceeded the sandbox-process cap")]; + } this.buffer = this.buffer.length === 0 ? incoming : Buffer.concat([this.buffer, incoming]); const results = []; for (;;) { @@ -1916,6 +1960,8 @@ class DuplexFrameDecoder { } results.push(decodeDuplexLine(line)); } + // Reconcile the local sandbox-process counter to the bytes still retained. + this.bytesInUse = this.buffer.length; return results; } }`; @@ -1959,6 +2005,12 @@ const responseTimeoutMs = Number( ); const maxQueueDepth = Number(process.env.PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH || "${DEFAULT_BRIDGE_MAX_QUEUE_DEPTH}"); const maxBodyBytes = Number(process.env.PAPERCLIP_BRIDGE_MAX_BODY_BYTES || "${DEFAULT_BRIDGE_MAX_BODY_BYTES}"); +// The host passes the separate sandbox-process raw-decoder cap here. The in-sandbox +// decoder enforces it locally under the "sandbox_process" scope; it never shares the +// host aggregate byte ledger. +const maxDuplexDecoderBytes = Number( + process.env.PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES || "${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}", +); const heartbeatIntervalMs = Number( process.env.PAPERCLIP_BRIDGE_HEARTBEAT_INTERVAL_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS}", ); @@ -2220,7 +2272,7 @@ function runDuplexGateway() { // local action here. } - const decoder = new DuplexFrameDecoder(); + const decoder = new DuplexFrameDecoder({ maxAggregateBytes: maxDuplexDecoderBytes }); process.stdin.on("data", (chunk) => { lastInboundAt = Date.now(); for (const result of decoder.push(chunk)) { diff --git a/packages/shared/src/telemetry/README.md b/packages/shared/src/telemetry/README.md index bba168e5cb7..9ff14488637 100644 --- a/packages/shared/src/telemetry/README.md +++ b/packages/shared/src/telemetry/README.md @@ -350,6 +350,24 @@ field. | `sandbox_duplex_loss_total` | One terminal duplex channel loss. The `loss_class` dimension records the phase. | | `sandbox_duplex_session_leak_total` | One leaked provider session at teardown. | +### Aggregate byte ledger metrics + +The host aggregate byte ledger owns one process-scoped gauge and two +process-scoped counters. The ledger bounds the retained bytes across every live +duplex route in one process. It sets the gauge on each reserve and each release. +It increments a counter on a rejected reservation and on an accounting defect. +These records carry no dimension label. The guarded counter store keys each +counter on `(companyId, metric)`, and the gauge reports one process value, so no +dynamic dimension rides them. The code owner is +`packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts`, and the metric +names are literal constants in `duplex-telemetry.ts`. + +| Metric | Type | Scope | +| --- | --- | --- | +| `sandbox_duplex_aggregate_bytes_in_use` | gauge | The aggregate retained bytes across every live duplex route. The ledger sets it on each reserve and each release. | +| `sandbox_duplex_aggregate_byte_reservation_rejections_total` | counter | One rejected aggregate byte reservation. The ledger increments it when a reservation would pass the aggregate ceiling. | +| `sandbox_duplex_aggregate_byte_accounting_underflow_total` | counter | One aggregate byte accounting defect. The ledger increments it on a double release or on a transfer of a token it does not hold. | + ### Dimension keys Counters carry no dimension labels. The guarded counter store keys each counter diff --git a/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts b/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts new file mode 100644 index 00000000000..a6f1a503484 --- /dev/null +++ b/server/src/__tests__/duplex-aggregate-ceiling-env.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; +import { resolveDuplexAggregateCeilingBytesFromEnv } from "../duplex-aggregate-ceiling-env.js"; + +describe("resolveDuplexAggregateCeilingBytesFromEnv", () => { + it("uses the safe default and reports nothing when the variable is absent", () => { + const onRejectedOverride = vi.fn(); + + const resolved = resolveDuplexAggregateCeilingBytesFromEnv(undefined, onRejectedOverride); + + expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + expect(onRejectedOverride).not.toHaveBeenCalled(); + }); + + it("treats a whitespace-only value as a present invalid override, not as absent", () => { + const rawOverride = " "; + const onRejectedOverride = vi.fn(); + + // The resolver must continue with the safe default. Startup does not fail. + const resolved = resolveDuplexAggregateCeilingBytesFromEnv(rawOverride, onRejectedOverride); + + expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + // The rejection reporter fires one time. A blank value is now visible as + // invalid instead of silent as absent. + expect(onRejectedOverride).toHaveBeenCalledTimes(1); + // The reporter receives only the parsed number. The raw whitespace string + // never reaches the reporter, so no operator-supplied text can reach a log + // line. + const reportedValue = onRejectedOverride.mock.calls[0]?.[0]; + expect(typeof reportedValue).toBe("number"); + expect(reportedValue).not.toBe(rawOverride); + expect(Number.isFinite(reportedValue)).toBe(true); + }); + + it("treats an empty-string value as a present invalid override", () => { + const onRejectedOverride = vi.fn(); + + const resolved = resolveDuplexAggregateCeilingBytesFromEnv("", onRejectedOverride); + + expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + expect(onRejectedOverride).toHaveBeenCalledTimes(1); + }); + + it("rejects a present non-numeric value and uses the safe default", () => { + const onRejectedOverride = vi.fn(); + + const resolved = resolveDuplexAggregateCeilingBytesFromEnv("not-a-number", onRejectedOverride); + + expect(resolved).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + expect(onRejectedOverride).toHaveBeenCalledTimes(1); + }); + + it("passes a valid override through unchanged", () => { + const onRejectedOverride = vi.fn(); + + const resolved = resolveDuplexAggregateCeilingBytesFromEnv("1048576", onRejectedOverride); + + expect(resolved).toBe(1048576); + expect(onRejectedOverride).not.toHaveBeenCalled(); + }); + + it("accepts a valid override that carries surrounding whitespace", () => { + const onRejectedOverride = vi.fn(); + + const resolved = resolveDuplexAggregateCeilingBytesFromEnv(" 1048576 ", onRejectedOverride); + + expect(resolved).toBe(1048576); + expect(onRejectedOverride).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/__tests__/environment-execution-target-duplex.test.ts b/server/src/__tests__/environment-execution-target-duplex.test.ts index 7af64eb431f..663653c6995 100644 --- a/server/src/__tests__/environment-execution-target-duplex.test.ts +++ b/server/src/__tests__/environment-execution-target-duplex.test.ts @@ -21,6 +21,11 @@ vi.mock("../services/plugin-environment-driver.js", async (importActual) => ({ })); import type { EffectiveSandboxCapabilities } from "@paperclipai/adapter-utils/execution-target"; +import { adapterExecutionTargetDuplexAggregateByteLedger } from "@paperclipai/adapter-utils/execution-target"; +import { + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + DuplexAggregateByteLedger, +} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { createSshCommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/ssh"; import type { Environment, EnvironmentLease } from "@paperclipai/shared"; import { resolveEnvironmentExecutionTarget } from "../services/environment-execution-target.js"; @@ -293,6 +298,50 @@ describe("EnvironmentRuntimeService.openDuplexChannel capability gate", () => { }); }); +describe("sandbox execution target aggregate byte ledger stamp", () => { + it("stamps the injected ledger on the target with object identity", async () => { + const ledger = new DuplexAggregateByteLedger({ + ceilingBytes: DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + }); + const target = await resolveEnvironmentExecutionTarget({ + db: {} as never, + companyId: "company-1", + adapterType: "codex_local", + environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } }, + leaseId: "lease-1", + leaseMetadata: { remoteCwd: "/work" }, + lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never, + duplexAggregateByteLedger: ledger, + }); + if (!target || target.kind !== "remote" || target.transport !== "sandbox") { + throw new Error("expected a sandbox execution target"); + } + // The stamped field and the accessor both return the one injected object, so + // one process-owned ledger reaches the host bridge seam. + expect(target.duplexAggregateByteLedger).toBe(ledger); + expect(adapterExecutionTargetDuplexAggregateByteLedger(target)).toBe(ledger); + }); + + it("leaves the ledger absent when the host injects none", async () => { + const target = await resolveEnvironmentExecutionTarget({ + db: {} as never, + companyId: "company-1", + adapterType: "codex_local", + environment: { id: "env-1", driver: "sandbox", config: { provider: "daytona" } }, + leaseId: "lease-1", + leaseMetadata: { remoteCwd: "/work" }, + lease: { id: "lease-1", leasePolicy: "reuse_by_environment" } as never, + }); + if (!target || target.kind !== "remote" || target.transport !== "sandbox") { + throw new Error("expected a sandbox execution target"); + } + // A run with no injected ledger keeps the seam null; the accessor never makes + // a fresh ledger. + expect(target.duplexAggregateByteLedger).toBeNull(); + expect(adapterExecutionTargetDuplexAggregateByteLedger(target)).toBeNull(); + }); +}); + // Build a sandbox execution target with a fixed capability snapshot and a fake // environment runtime whose openDuplexChannel is a spy. The helper returns the // runner and the spy so a test reads the capability-gated member. diff --git a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs index 013ecb03629..69ac9e7e452 100644 --- a/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs +++ b/server/src/__tests__/fixtures/plugin-worker-duplex-channel.cjs @@ -23,6 +23,18 @@ // and never lets the exit crowd a data frame out of the pre-open hold. // - `echoInput`: when true, the fixture echoes each `duplexChannelWrite` back as // one data notification for the bound session. +// - `writeReplyDelayMs`: when a positive number, the fixture delays each +// `duplexChannelWrite` reply by that many milliseconds, so a test proves the +// host holds the pending-write reservation until the RPC settles. +// - `stopReadingStdinAfterOpen`: when true, the fixture stops reading its stdin +// right after it sends the open reply. The host writes then stay in the host +// stdin write buffer, so a test proves the host meters the transport buffer and +// never releases the transport token on the write-RPC timeout. +// - `exitAfterStopMs`: when a positive number, and the fixture stopped reading +// stdin, the fixture exits after that many milliseconds. The worker exit +// discards the stdin buffer, so a test proves the host releases every held +// transport token on the worker exit. It gives a fast exit for a worker that no +// longer reads its stdin and never sees the shutdown request. // - `closeMode`: "ack" | "bad-ack" | "no-ack" (default "ack"). It controls the // close reply, so a test proves the host retires the worker on an unconfirmed // close. @@ -134,6 +146,8 @@ rl.on("line", (line) => { closeMode, echoInput: directive.echoInput === true, noWriteReply: mode === "no-write-reply", + writeReplyDelayMs: + typeof directive.writeReplyDelayMs === "number" ? directive.writeReplyDelayMs : 0, emitAfterCloseChunk: typeof directive.emitAfterCloseChunk === "string" ? directive.emitAfterCloseChunk : null, }); @@ -170,6 +184,20 @@ rl.on("line", (line) => { reply(); } + if (directive.stopReadingStdinAfterOpen === true) { + // Stop reading stdin, so every later host write stays in the host stdin + // write buffer. The host meters the transport buffer and holds the transport + // token until the stream flush, the stream error, or the worker exit. + rl.pause(); + process.stdin.pause(); + if (typeof directive.exitAfterStopMs === "number" && directive.exitAfterStopMs >= 0) { + // Exit after the delay, so a test observes the worker exit release the held + // transport tokens without a slow shutdown drain on an unread stdin. + setTimeout(() => process.exit(0), directive.exitAfterStopMs); + } + return; + } + // Emit the scripted data and the exit after the open reply, so the host // binds the route first. Each frame echoes the exact pair; a test overrides // `sid` or `rid` to force a mismatch. @@ -205,7 +233,14 @@ rl.on("line", (line) => { }, }); } - send({ jsonrpc: "2.0", id: message.id, result: null }); + const replyWrite = () => send({ jsonrpc: "2.0", id: message.id, result: null }); + if (entry.writeReplyDelayMs > 0) { + // Delay the write reply, so the host holds the pending-write reservation for + // a measurable time before the RPC settles. + setTimeout(replyWrite, entry.writeReplyDelayMs); + } else { + replyWrite(); + } return; } diff --git a/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts new file mode 100644 index 00000000000..2b2215891a9 --- /dev/null +++ b/server/src/__tests__/plugin-worker-manager-duplex-byte-ledger.test.ts @@ -0,0 +1,348 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, vi } from "vitest"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import { + DuplexAggregateByteLedger, + type DuplexAggregateByteLedgerTelemetry, +} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; +import { + createDuplexRouteSlotController, + createPluginWorkerHandle, +} from "../services/plugin-worker-manager.js"; + +// This suite proves the plugin worker manager charges every retained duplex route +// representation against the injected aggregate byte ledger, and releases each +// token exactly once through the one cleanup path. The tests drive a real worker +// fixture process, so they exercise the true frame order across the bind boundary. + +const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"); +const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join( + FIXTURES_DIR, + "plugin-worker-duplex-channel.cjs", +); + +const TEST_MANIFEST: PaperclipPluginManifestV1 = { + id: "test.plugin", + apiVersion: 1, + version: "1.0.0", + displayName: "Test plugin", + description: "Test plugin", + author: "Paperclip", + categories: ["automation"], + capabilities: [], + entrypoints: { worker: "dist/worker.js" }, +}; + +function makeDuplexHandle(extra?: Record) { + return createPluginWorkerHandle("test.plugin", { + entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT, + manifest: TEST_MANIFEST, + config: {}, + instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, + apiVersion: 1, + hostHandlers: {}, + ...extra, + }); +} + +// The test directive rides in `providerLeaseId`, an opaque field the manager +// forwards to the worker unchanged. +function duplexOpenInput(directive: unknown, companyId = "company-1") { + return { + driverKey: "daytona", + companyId, + environmentId: "env-1", + providerLeaseId: JSON.stringify(directive), + command: "bridge-callback", + }; +} + +// A counting telemetry surface. It records how many times the ledger rejected a +// reservation and how many accounting defects it saw, so a test asserts a +// fail-closed rejection and proves the cleanup never double-releases. +function countingTelemetry(): DuplexAggregateByteLedgerTelemetry & { + rejections: number; + underflows: number; +} { + const state = { + rejections: 0, + underflows: 0, + setBytesInUse() {}, + recordReservationRejection() { + state.rejections += 1; + }, + recordAccountingUnderflow() { + state.underflows += 1; + }, + }; + return state; +} + +describe("plugin worker manager duplex aggregate byte ledger", () => { + it("keeps terminalized buffered bytes charged until a late listener drains them, then a worker exit is harmless", async () => { + const telemetry = countingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); + // Lower the per-chunk char bound, so the second chunk ends the route while the + // first chunk stays buffered. + const handle = makeDuplexHandle({ + duplexAggregateByteLedger: ledger, + duplexChannelLimits: { maxChunkChars: 4 }, + }); + try { + await handle.start(); + const session = await handle.openDuplexChannel( + // No listener attaches, so "ok" buffers and charges its two raw bytes. The + // "toolong" chunk passes the per-chunk bound, so the route terminalizes and + // moves the still-buffered "ok" token to the terminal registry. + duplexOpenInput({ data: [{ chunk: "ok" }, { chunk: "toolong" }] }), + ); + // The buffered "ok" stays charged after the route leaves the live map. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(2); + expect(ledger.liveTokenCount).toBe(1); + }); + // A late listener drains the terminal buffered record and releases its token. + const chunks: string[] = []; + session.onData((chunk) => chunks.push(chunk)); + await vi.waitFor(() => { + expect(chunks).toEqual(["ok"]); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + // The over-bound chunk retained nothing, so no reservation ever rejected. + expect(telemetry.rejections).toBe(0); + } finally { + await handle.stop().catch(() => undefined); + } + // The worker exit sweep runs on stop. The ledger stays at zero, and the sweep + // records no accounting defect, so the drain and the exit never double-release. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(telemetry.underflows).toBe(0); + }); + + it("transfers pre-bind held bytes to the buffered representation across the bind, then releases them on drain", async () => { + const telemetry = countingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handle.start(); + const session = await handle.openDuplexChannel( + // The worker writes the open reply and the two data frames in one stdout + // write, so both frames arrive before the route binds. The host holds them + // as pre-bind events, then the bind replays and transfers each token to the + // buffered representation. + duplexOpenInput({ batchWithOpenReply: true, data: [{ chunk: "aa" }, { chunk: "bb" }] }), + ); + // The two held events keep their exact reserved bytes across the transfer, so + // the live-token count never grows and the gauge never re-admits. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(4); + expect(ledger.liveTokenCount).toBe(2); + }); + const chunks: string[] = []; + session.onData((chunk) => chunks.push(chunk)); + await vi.waitFor(() => { + expect(chunks).toEqual(["aa", "bb"]); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + } finally { + await handle.stop().catch(() => undefined); + } + expect(ledger.bytesInUse).toBe(0); + expect(telemetry.underflows).toBe(0); + }); + + it("fails closed and retains nothing when a buffered reservation would pass the ceiling", async () => { + const telemetry = countingTelemetry(); + // A four-byte ceiling. One five-byte chunk cannot fit. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handle.start(); + await handle.openDuplexChannel( + // No listener attaches, so "hello" tries to buffer. Its five raw bytes pass + // the four-byte ceiling, so the reservation rejects and the route ends. + duplexOpenInput({ data: [{ chunk: "hello" }] }), + ); + await vi.waitFor(() => { + expect(telemetry.rejections).toBeGreaterThanOrEqual(1); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + } finally { + await handle.stop().catch(() => undefined); + } + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(telemetry.underflows).toBe(0); + }); +}); + +// A telemetry surface that also tracks the peak gauge value. A test asserts the +// peak never passes the ceiling, so it proves aggregate admission stops at the +// ceiling instead of overshooting it. +function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & { + gauge: number; + peak: number; + rejections: number; + underflows: number; +} { + const state = { + gauge: 0, + peak: 0, + rejections: 0, + underflows: 0, + setBytesInUse(bytes: number) { + state.gauge = bytes; + if (bytes > state.peak) state.peak = bytes; + }, + recordReservationRejection() { + state.rejections += 1; + }, + recordAccountingUnderflow() { + state.underflows += 1; + }, + }; + return state; +} + +// The exact raw byte count of one buffered chunk each route retains. +const LOAD_CHUNK_BYTES = 8; +const LOAD_CHUNK = "x".repeat(LOAD_CHUNK_BYTES); + +describe("plugin worker manager duplex aggregate byte ledger load across workers", () => { + it("stops aggregate byte admission at the ceiling across many routes and workers while the route-count controller stays open", async () => { + const telemetry = peakTrackingTelemetry(); + // The ceiling holds exactly six eight-byte buffered chunks. + const fittingRoutes = 6; + const ceilingBytes = fittingRoutes * LOAD_CHUNK_BYTES; + const ledger = new DuplexAggregateByteLedger({ ceilingBytes, telemetry }); + // The route-count controller has ample room, so the byte ledger binds first. + const routeSlots = createDuplexRouteSlotController(100); + const handleCount = 3; + const routesPerHandle = 3; + const totalRoutes = handleCount * routesPerHandle; + const rejectedRoutes = totalRoutes - fittingRoutes; + const handles: Array> = []; + try { + for (let h = 0; h < handleCount; h += 1) { + const handle = makeDuplexHandle({ + duplexAggregateByteLedger: ledger, + duplexRouteSlots: routeSlots, + }); + await handle.start(); + handles.push(handle); + } + // Open every route in order. Each worker batches one data chunk with the open + // reply, so the host holds the chunk as a pre-bind event and reserves its exact + // raw bytes before the route binds. No listener attaches, so a bound route holds + // the chunk. The first six chunks fit the ceiling. Each later reservation passes + // the ceiling, so the ledger rejects it and the route fails closed with the open + // marker (not the route-busy marker), because the route-count controller has room. + let opened = 0; + let failedClosed = 0; + for (let h = 0; h < handleCount; h += 1) { + for (let r = 0; r < routesPerHandle; r += 1) { + try { + await handles[h].openDuplexChannel( + duplexOpenInput({ + workerSessionId: `ws-${h}-${r}`, + batchWithOpenReply: true, + data: [{ chunk: LOAD_CHUNK }], + }), + ); + opened += 1; + } catch (err) { + // A byte-ceiling rejection fails the route closed with the open marker. + // The route-count controller had room, so this is never the route-busy + // marker. + expect(String(err)).toContain("DUPLEX_CHANNEL_OPEN_FAILED"); + expect(String(err)).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); + failedClosed += 1; + } + } + } + // Six routes bound and hold their bytes; three failed closed at the ceiling. + expect(opened).toBe(fittingRoutes); + expect(failedClosed).toBe(rejectedRoutes); + expect(opened + failedClosed).toBe(totalRoutes); + // The aggregate gauge holds at the ceiling and the ledger recorded one rejection + // per over-ceiling route. + await vi.waitFor(() => { + expect(telemetry.rejections).toBe(rejectedRoutes); + expect(ledger.bytesInUse).toBe(ceilingBytes); + expect(ledger.liveTokenCount).toBe(fittingRoutes); + }); + // The gauge never passed the ceiling, so admission stopped at the ceiling and + // never overshot it. + expect(telemetry.peak).toBe(ceilingBytes); + } finally { + for (const handle of handles) await handle.stop().catch(() => undefined); + } + // Every worker exit sweep released its buffered tokens once, so the ledger ends + // at zero with no accounting defect. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(telemetry.underflows).toBe(0); + }); + + it("still bounds the route count for fixed per-route overhead when the byte ceiling has ample room", async () => { + const telemetry = peakTrackingTelemetry(); + // A large byte ceiling, so retained bytes never bind admission. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); + // The route-count controller holds two slots for the whole process. + const routeSlots = createDuplexRouteSlotController(2); + const handleA = makeDuplexHandle({ + duplexAggregateByteLedger: ledger, + duplexRouteSlots: routeSlots, + }); + const handleB = makeDuplexHandle({ + duplexAggregateByteLedger: ledger, + duplexRouteSlots: routeSlots, + }); + let opened = 0; + let busy = 0; + try { + await handleA.start(); + await handleB.start(); + // Attempt four routes across the two workers. The shared controller admits the + // first two and rejects the last two with the fixed route-busy error. + const attempts: Array<[ReturnType, string]> = [ + [handleA, "ws-a1"], + [handleA, "ws-a2"], + [handleB, "ws-b1"], + [handleB, "ws-b2"], + ]; + for (const [handle, ws] of attempts) { + try { + await handle.openDuplexChannel( + duplexOpenInput({ workerSessionId: ws, data: [{ chunk: "aa" }] }), + ); + opened += 1; + } catch (err) { + expect(String(err)).toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); + busy += 1; + } + } + // The route-count controller admitted two routes and rejected the rest, so it + // still bounds the fixed per-route overhead on its own. + expect(opened).toBe(2); + expect(busy).toBe(2); + // The two open routes buffered four raw bytes, far under the byte ceiling, and + // the byte ledger rejected nothing. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(4); + expect(ledger.liveTokenCount).toBe(2); + }); + expect(telemetry.rejections).toBe(0); + } finally { + await handleA.stop().catch(() => undefined); + await handleB.stop().catch(() => undefined); + } + expect(ledger.bytesInUse).toBe(0); + expect(telemetry.underflows).toBe(0); + }); +}); diff --git a/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts new file mode 100644 index 00000000000..4c22219eba7 --- /dev/null +++ b/server/src/__tests__/plugin-worker-manager-duplex-pending-write-ledger.test.ts @@ -0,0 +1,251 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, vi } from "vitest"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + DuplexAggregateByteLedger, + type DuplexAggregateByteLedgerTelemetry, +} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; + +// Mock the shared logger, so a test reads the fixed rejection marker the manager +// logs when a pending-write reservation fails. The child logger returns the same +// object, so `log.warn` is this mock's `warn`. +vi.mock("../middleware/logger.js", () => { + const mockLogger: Record = { + trace: vi.fn(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + fatal: vi.fn(), + child: vi.fn(() => mockLogger), + }; + return { logger: mockLogger, httpLogger: vi.fn() }; +}); + +import { logger } from "../middleware/logger.js"; +import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js"; + +// This suite proves the plugin worker manager charges every host→worker write raw +// payload against the injected aggregate byte ledger under the `pending_write` +// owner, and releases each token one time when the write RPC settles. The child +// reads its stdin here, so the separate `stdin_write` transport token flushes and +// releases at once. Each test waits for that flush, so the assertions isolate the +// raw-payload token. The transport token has its own suite. + +const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"); +const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join( + FIXTURES_DIR, + "plugin-worker-duplex-channel.cjs", +); + +const TEST_MANIFEST: PaperclipPluginManifestV1 = { + id: "test.plugin", + apiVersion: 1, + version: "1.0.0", + displayName: "Test plugin", + description: "Test plugin", + author: "Paperclip", + categories: ["automation"], + capabilities: [], + entrypoints: { worker: "dist/worker.js" }, +}; + +function makeDuplexHandle(extra?: Record) { + return createPluginWorkerHandle("test.plugin", { + entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT, + manifest: TEST_MANIFEST, + config: {}, + instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, + apiVersion: 1, + hostHandlers: {}, + ...extra, + }); +} + +// The test directive rides in `providerLeaseId`, an opaque field the manager +// forwards to the worker unchanged. +function duplexOpenInput(directive: unknown, companyId = "company-1") { + return { + driverKey: "daytona", + companyId, + environmentId: "env-1", + providerLeaseId: JSON.stringify(directive), + command: "bridge-callback", + }; +} + +// A telemetry surface that tracks the peak gauge value and the two counters. A +// test asserts the peak never passes the ceiling, so it proves aggregate admission +// stops at the ceiling instead of overshooting it. +function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & { + gauge: number; + peak: number; + rejections: number; + underflows: number; +} { + const state = { + gauge: 0, + peak: 0, + rejections: 0, + underflows: 0, + setBytesInUse(bytes: number) { + state.gauge = bytes; + if (bytes > state.peak) state.peak = bytes; + }, + recordReservationRejection() { + state.rejections += 1; + }, + recordAccountingUnderflow() { + state.underflows += 1; + }, + }; + return state; +} + +// Return every reason string the manager logged through the mock `warn` sink. +function loggedWarnReasons(): string[] { + const calls = (logger.warn as unknown as { mock: { calls: unknown[][] } }).mock.calls; + return calls + .map((call) => { + const first = call[0]; + return first && typeof first === "object" + ? (first as { reason?: unknown }).reason + : undefined; + }) + .filter((reason): reason is string => typeof reason === "string"); +} + +describe("plugin worker manager duplex pending-write byte ledger", () => { + it("charges each held raw payload and returns to zero after worker exit", async () => { + const telemetry = peakTrackingTelemetry(); + // The ceiling has ample room, so a transient transport reservation never + // rejects. Each write holds its raw payload until the RPC settles. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + const writeBytes = 1000; + const writeData = "x".repeat(writeBytes); + try { + await handle.start(); + // The worker reads its stdin but never replies to a write, so each write RPC + // stays pending and holds its raw payload. + const route = await handle.openDuplexChannel( + duplexOpenInput({ workerSessionId: "ws-a", mode: "no-write-reply" }), + ); + route.write(writeData); + route.write(writeData); + route.write(writeData); + // The worker reads its stdin, so each transport token flushes and releases. + // Only the three held raw payloads remain, so the gauge settles at three + // times the payload byte count with three live tokens. + await vi.waitFor(() => { + expect(ledger.liveTokenCount).toBe(3); + expect(ledger.bytesInUse).toBe(3 * writeBytes); + }); + expect(telemetry.peak).toBeLessThanOrEqual(ledger.ceilingBytes); + } finally { + await handle.stop().catch(() => undefined); + } + // The worker exit settles each pending write, so each token releases one time + // and the ledger ends at zero with no accounting defect. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + expect(telemetry.underflows).toBe(0); + }); + + it("releases the raw-payload token after a delayed write RPC settles, with no worker exit", async () => { + const telemetry = peakTrackingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handle.start(); + // The worker reads its stdin and delays its write reply, so the host holds the + // raw-payload reservation for a measurable time before the RPC settles. + const route = await handle.openDuplexChannel( + duplexOpenInput({ workerSessionId: "ws-d", writeReplyDelayMs: 150 }), + ); + const writeBytes = 1000; + route.write("x".repeat(writeBytes)); + // The transport token flushes at once, so one raw-payload token stays held for + // the full write byte count while the RPC is in flight. + await vi.waitFor(() => { + expect(ledger.liveTokenCount).toBe(1); + expect(ledger.bytesInUse).toBe(writeBytes); + }); + // The delayed reply settles the RPC, and the `finally` releases the token. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + } finally { + await handle.stop().catch(() => undefined); + } + expect(ledger.bytesInUse).toBe(0); + expect(telemetry.underflows).toBe(0); + }); + + it("reserves the exact UTF-8 byte count of the raw payload, not the character length", async () => { + const telemetry = peakTrackingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1 << 20, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handle.start(); + const route = await handle.openDuplexChannel( + duplexOpenInput({ workerSessionId: "ws-u", mode: "no-write-reply" }), + ); + // "a€" is two characters but four UTF-8 bytes (one plus three). The raw-payload + // reservation must charge four bytes, so it uses the UTF-8 byte count. + const data = "a€"; + expect(data.length).toBe(2); + expect(Buffer.byteLength(data, "utf8")).toBe(4); + route.write(data); + // The transport token flushes, so one raw-payload token of four bytes remains. + await vi.waitFor(() => { + expect(ledger.liveTokenCount).toBe(1); + expect(ledger.bytesInUse).toBe(4); + }); + } finally { + await handle.stop().catch(() => undefined); + } + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + expect(telemetry.underflows).toBe(0); + }); + + it("ends an over-ceiling single write fail-closed with the aggregate marker, not the route-busy marker", async () => { + vi.mocked(logger.warn).mockClear(); + const telemetry = peakTrackingTelemetry(); + // The ceiling is smaller than one raw payload, so the raw-payload reservation + // fails before the frame reaches the transport. The manager retains nothing. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 100, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handle.start(); + const route = await handle.openDuplexChannel( + duplexOpenInput({ workerSessionId: "ws-o", mode: "no-write-reply" }), + ); + // The single write is larger than the ceiling. The raw-payload reservation + // fails, so the manager ends the route with the aggregate marker, never the + // route-busy marker, and never writes the frame. + route.write("x".repeat(200)); + expect(telemetry.rejections).toBeGreaterThanOrEqual(1); + expect(ledger.bytesInUse).toBe(0); + expect(telemetry.peak).toBeLessThanOrEqual(ledger.ceilingBytes); + const reasons = loggedWarnReasons(); + expect(reasons).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + expect(reasons).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); + } finally { + await handle.stop().catch(() => undefined); + } + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + expect(telemetry.underflows).toBe(0); + }); +}); diff --git a/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts b/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts new file mode 100644 index 00000000000..27668aafd26 --- /dev/null +++ b/server/src/__tests__/plugin-worker-manager-duplex-stdin-write-ledger.test.ts @@ -0,0 +1,297 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it, vi } from "vitest"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + DuplexAggregateByteLedger, + type DuplexAggregateByteLedgerTelemetry, +} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; + +// Mock the shared logger, so a test reads the fixed rejection marker the manager +// logs when a transport reservation fails. The child logger returns the same +// object, so `log.warn` is this mock's `warn`. +vi.mock("../middleware/logger.js", () => { + const mockLogger: Record = { + trace: vi.fn(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + fatal: vi.fn(), + child: vi.fn(() => mockLogger), + }; + return { logger: mockLogger, httpLogger: vi.fn() }; +}); + +import { logger } from "../middleware/logger.js"; +import { createPluginWorkerHandle } from "../services/plugin-worker-manager.js"; + +// This suite proves the plugin worker manager charges the child-stdin transport +// buffer for every host→worker duplex write, under the `stdin_write` owner. The +// worker completes the open, then stops reading its stdin. The host write then +// stays in the host stdin write buffer. The tests prove the reservation covers the +// serialized frame, holds until the flush or the worker exit, survives the write +// RPC timeout and the route terminalization, and bounds the aggregate transport +// bytes across several routes and workers. + +const FIXTURES_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "fixtures"); +const DUPLEX_CHANNEL_WORKER_ENTRYPOINT = path.join( + FIXTURES_DIR, + "plugin-worker-duplex-channel.cjs", +); + +const TEST_MANIFEST: PaperclipPluginManifestV1 = { + id: "test.plugin", + apiVersion: 1, + version: "1.0.0", + displayName: "Test plugin", + description: "Test plugin", + author: "Paperclip", + categories: ["automation"], + capabilities: [], + entrypoints: { worker: "dist/worker.js" }, +}; + +function makeDuplexHandle(extra?: Record) { + return createPluginWorkerHandle("test.plugin", { + entrypointPath: DUPLEX_CHANNEL_WORKER_ENTRYPOINT, + manifest: TEST_MANIFEST, + config: {}, + instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, + apiVersion: 1, + hostHandlers: {}, + ...extra, + }); +} + +// The test directive rides in `providerLeaseId`, an opaque field the manager +// forwards to the worker unchanged. +function duplexOpenInput(directive: unknown, companyId = "company-1") { + return { + driverKey: "daytona", + companyId, + environmentId: "env-1", + providerLeaseId: JSON.stringify(directive), + command: "bridge-callback", + }; +} + +// A telemetry surface that tracks the peak gauge value and the two counters. A +// test asserts the peak never passes the ceiling, so it proves aggregate admission +// stops at the ceiling instead of overshooting it. +function peakTrackingTelemetry(): DuplexAggregateByteLedgerTelemetry & { + gauge: number; + peak: number; + rejections: number; + underflows: number; +} { + const state = { + gauge: 0, + peak: 0, + rejections: 0, + underflows: 0, + setBytesInUse(bytes: number) { + state.gauge = bytes; + if (bytes > state.peak) state.peak = bytes; + }, + recordReservationRejection() { + state.rejections += 1; + }, + recordAccountingUnderflow() { + state.underflows += 1; + }, + }; + return state; +} + +// Return every reason string the manager logged through the mock `warn` sink. +function loggedWarnReasons(): string[] { + const calls = (logger.warn as unknown as { mock: { calls: unknown[][] } }).mock.calls; + return calls + .map((call) => { + const first = call[0]; + return first && typeof first === "object" + ? (first as { reason?: unknown }).reason + : undefined; + }) + .filter((reason): reason is string => typeof reason === "string"); +} + +// A frame the host keeps in its stdin write buffer must pass the operating-system +// pipe buffer (about 64 KiB), so the write callback stays pending while the child +// does not read. Each write here is larger than that, so the host holds the +// transport token until the worker exit. +const HELD_WRITE_CHARS = 200_000; + +describe("plugin worker manager duplex stdin transport byte ledger", () => { + it("reserves the encoded serialized-frame size, larger than the raw payload, including JSON escaping", async () => { + const telemetry = peakTrackingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handle.start(); + const route = await handle.openDuplexChannel( + duplexOpenInput({ + workerSessionId: "ws-a", + stopReadingStdinAfterOpen: true, + exitAfterStopMs: 500, + }), + ); + // Every character is a quote. The JSON serialization escapes each quote to two + // bytes, so the serialized frame is at least twice the raw payload byte count. + const data = '"'.repeat(50_000); + const rawBytes = Buffer.byteLength(data, "utf8"); + route.write(data); + // The raw-payload token and the transport token both hold at once. The worker + // does not read its stdin, so the transport token never flushes. + expect(ledger.liveTokenCount).toBe(2); + const transportBytes = ledger.bytesInUse - rawBytes; + // The transport reservation covers the serialized frame, so it is larger than + // the raw payload and it includes the doubled escaped quotes. + expect(transportBytes).toBeGreaterThan(rawBytes); + expect(transportBytes).toBeGreaterThanOrEqual(2 * rawBytes); + } finally { + await handle.stop().catch(() => undefined); + } + // The worker exit discards the stdin buffer, so every token releases one time. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + expect(telemetry.underflows).toBe(0); + }); + + it("holds the transport token across a write-RPC timeout and releases it on worker exit", async () => { + const telemetry = peakTrackingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry }); + // The write RPC times out fast; the worker exits later. The gap lets a test + // prove the timeout does not release the transport token. + const handle = makeDuplexHandle({ + duplexAggregateByteLedger: ledger, + duplexChannelLimits: { openTimeoutMs: 150 }, + }); + try { + await handle.start(); + const route = await handle.openDuplexChannel( + duplexOpenInput({ + workerSessionId: "ws-t", + stopReadingStdinAfterOpen: true, + exitAfterStopMs: 900, + }), + ); + const data = "x".repeat(HELD_WRITE_CHARS); + const rawBytes = Buffer.byteLength(data, "utf8"); + route.write(data); + // Both tokens hold at first: the raw payload and the serialized frame. + expect(ledger.liveTokenCount).toBe(2); + const transportBytes = ledger.bytesInUse - rawBytes; + expect(transportBytes).toBeGreaterThan(0); + // The write RPC times out. The RPC settle releases the raw-payload token, so + // one token remains: the transport token. The timeout never releases it. + await vi.waitFor(() => { + expect(ledger.liveTokenCount).toBe(1); + }); + expect(ledger.bytesInUse).toBe(transportBytes); + } finally { + await handle.stop().catch(() => undefined); + } + // The worker exit releases the still-held transport token. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + expect(telemetry.underflows).toBe(0); + }); + + it("does not release the transport token when the route terminalizes", async () => { + const telemetry = peakTrackingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 << 20, telemetry }); + const handle = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handle.start(); + const route = await handle.openDuplexChannel( + duplexOpenInput({ + workerSessionId: "ws-c", + stopReadingStdinAfterOpen: true, + exitAfterStopMs: 700, + }), + ); + const data = "x".repeat(HELD_WRITE_CHARS); + const rawBytes = Buffer.byteLength(data, "utf8"); + route.write(data); + expect(ledger.liveTokenCount).toBe(2); + const transportBytes = ledger.bytesInUse - rawBytes; + // End the route. Route terminalization releases the route-owned tokens, but it + // must never release the transport token, so at least the serialized frame + // stays charged. + void route.close(); + expect(ledger.bytesInUse).toBeGreaterThanOrEqual(transportBytes); + } finally { + await handle.stop().catch(() => undefined); + } + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + expect(telemetry.underflows).toBe(0); + }); + + it("bounds the aggregate transport bytes across several routes and workers, ending an over-ceiling near-limit write with the aggregate marker", async () => { + vi.mocked(logger.warn).mockClear(); + const telemetry = peakTrackingTelemetry(); + // The ceiling admits only a couple of held serialized frames, so a near-limit + // write across the routes passes it and rejects. + const ceilingBytes = 600_000; + const ledger = new DuplexAggregateByteLedger({ ceilingBytes, telemetry }); + const handleA = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + const handleB = makeDuplexHandle({ duplexAggregateByteLedger: ledger }); + try { + await handleA.start(); + await handleB.start(); + const routeA = await handleA.openDuplexChannel( + duplexOpenInput({ + workerSessionId: "ws-a", + stopReadingStdinAfterOpen: true, + exitAfterStopMs: 900, + }), + ); + const routeB = await handleB.openDuplexChannel( + duplexOpenInput({ + workerSessionId: "ws-b", + stopReadingStdinAfterOpen: true, + exitAfterStopMs: 900, + }), + ); + // Each write mixes escaped quotes and multi-byte characters near the per-write + // size. The serialized frame escapes each quote and keeps each euro sign as + // three UTF-8 bytes, so the reservation must count the encoded size. + const escaped = '"'.repeat(40_000); + const multiByte = "€".repeat(20_000); + const nearLimit = escaped + multiByte; + const routes = [routeA, routeB, routeA, routeB]; + for (const route of routes) { + route.write(nearLimit); + // The aggregate bytes never pass the ceiling. Admission stops at the ceiling. + expect(ledger.bytesInUse).toBeLessThanOrEqual(ceilingBytes); + } + // At least one near-limit write passed the ceiling and rejected. The manager + // ended that route with the aggregate marker, never the route-busy marker. + expect(telemetry.rejections).toBeGreaterThanOrEqual(1); + expect(telemetry.peak).toBeLessThanOrEqual(ceilingBytes); + const reasons = loggedWarnReasons(); + expect(reasons).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + expect(reasons).not.toContain("DUPLEX_CHANNEL_ROUTE_BUSY"); + } finally { + await handleA.stop().catch(() => undefined); + await handleB.stop().catch(() => undefined); + } + // Every worker exit releases its held transport tokens, so the ledger ends at + // zero with no accounting defect. + await vi.waitFor(() => { + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + expect(telemetry.underflows).toBe(0); + }); +}); diff --git a/server/src/duplex-aggregate-ceiling-env.ts b/server/src/duplex-aggregate-ceiling-env.ts new file mode 100644 index 00000000000..a1381c910cc --- /dev/null +++ b/server/src/duplex-aggregate-ceiling-env.ts @@ -0,0 +1,32 @@ +import { + resolveDuplexAggregateCeilingBytes, + type DuplexAggregateCeilingOverrideRejectionReporter, +} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; + +/** + * Resolve the aggregate duplex route byte ceiling from the raw operator override + * string. The host reads the override from + * PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES. + * + * This helper distinguishes an absent variable from a present blank value. An + * absent variable is `undefined`. The helper then uses the documented default and + * does not report a rejection. A present variable is a string, even when the + * string holds only whitespace. The helper sends every present string through + * `Number`, so a blank or whitespace-only value becomes `0`. The numeric + * validation in {@link resolveDuplexAggregateCeilingBytes} rejects `0`, reports it + * through `onRejectedOverride`, and returns the safe default. So a present blank + * value is visible as invalid instead of silent as absent. + * + * `Number` ignores surrounding whitespace, so a valid value with surrounding + * spaces still parses. The reporter receives only the parsed number, never the raw + * string, so no operator-supplied text reaches a log line. + */ +export function resolveDuplexAggregateCeilingBytesFromEnv( + rawOverride: string | undefined, + onRejectedOverride?: DuplexAggregateCeilingOverrideRejectionReporter, +): number { + return resolveDuplexAggregateCeilingBytes( + rawOverride === undefined ? undefined : Number(rawOverride), + onRejectedOverride, + ); +} diff --git a/server/src/index.ts b/server/src/index.ts index 6786407a65d..058348ddc30 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -83,6 +83,13 @@ import { createFeedbackTraceShareClientFromConfig } from "./services/feedback-sh import { buildRuntimeApiCandidateUrls, choosePrimaryRuntimeApiUrl } from "./runtime-api.js"; import { isLoopbackHost, rewriteLoopbackUrlPort } from "./url-utils.js"; import { createPluginWorkerManager } from "./services/plugin-worker-manager.js"; +import { + createDuplexAggregateByteLedgerTelemetry, + DuplexAggregateByteLedger, + type DuplexAggregateByteLedgerMetricSink, +} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; +import { resolveDuplexAggregateCeilingBytesFromEnv } from "./duplex-aggregate-ceiling-env.js"; +import { DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL } from "@paperclipai/adapter-utils/duplex-telemetry"; import { createStorageServiceFromConfig } from "./storage/index.js"; import { printStartupBanner } from "./startup-banner.js"; import { getBoardClaimWarningUrl, initializeBoardClaimChallenge } from "./board-claim.js"; @@ -754,9 +761,57 @@ export async function startServer(): Promise { databaseBackupInFlight = false; } }; - const pluginWorkerManager = createPluginWorkerManager(); + // The process-owned aggregate byte ledger for the sandbox duplex channel. One + // ledger per host process bounds the aggregate bytes that all live duplex routes + // retain. The route-count controller bounds only the route count, so without this + // ledger the per-route byte bounds multiply to many gigabytes at the maximum + // route count. The manager injects this same object into every worker handle, so + // one shared gauge bounds every host-side retention site. + // + // The optional operator override reads PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES. + // An absent variable uses the documented default. A present invalid, blank, + // whitespace-only, non-finite, zero, negative, non-integer, unsafe, or + // over-maximum value does not fail startup. The host process is multi-tenant, so + // one invalid environment value must not brick the whole host. The helper sends + // the raw string to the resolver, so a present blank value is invalid, not + // absent. The resolver rejects the invalid override and returns the safe default. + // The reporter logs the rejection loudly at error, so the misconfiguration stays + // visible while the host stays up. The log line carries only the rejected numeric + // value, never the raw string. + const duplexAggregateCeilingBytes = resolveDuplexAggregateCeilingBytesFromEnv( + process.env.PAPERCLIP_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + (rejectedValue) => { + logger.error( + { rejectedValue }, + "duplex aggregate byte ceiling override rejected; using the safe default", + ); + }, + ); + // The server has no process metric pipeline yet, so the ledger telemetry maps to + // the structured logger. The gauge logs at debug. A reservation rejection logs at + // warn, because it marks an availability limit hit. An accounting-underflow defect + // logs at error, because it marks a real cleanup bug. Each record carries only the + // fixed metric name and the numeric value; no route, company, run, or payload + // value reaches a log line. + const duplexAggregateByteLedgerMetricSink: DuplexAggregateByteLedgerMetricSink = { + setGauge(name, value) { + logger.debug({ metric: name, value }, "duplex aggregate byte ledger gauge"); + }, + incrementCounter(name) { + if (name === DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL) { + logger.error({ metric: name }, "duplex aggregate byte ledger accounting underflow"); + return; + } + logger.warn({ metric: name }, "duplex aggregate byte ledger reservation rejected"); + }, + }; + const duplexAggregateByteLedger = new DuplexAggregateByteLedger({ + ceilingBytes: duplexAggregateCeilingBytes, + telemetry: createDuplexAggregateByteLedgerTelemetry(duplexAggregateByteLedgerMetricSink), + }); + const pluginWorkerManager = createPluginWorkerManager({ duplexAggregateByteLedger }); const heartbeat = config.heartbeatSchedulerEnabled - ? heartbeatService(db as any, { pluginWorkerManager }) + ? heartbeatService(db as any, { pluginWorkerManager, duplexAggregateByteLedger }) : null; const decisionServiceOptions = { wakeOriginAgent: createDecisionWakeOriginAgent(heartbeat?.wakeup ?? null), diff --git a/server/src/services/environment-execution-target.ts b/server/src/services/environment-execution-target.ts index 90fc7ac3c9b..c792929f507 100644 --- a/server/src/services/environment-execution-target.ts +++ b/server/src/services/environment-execution-target.ts @@ -6,6 +6,7 @@ import { type AdapterExecutionTarget, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexTelemetryRecorder } from "@paperclipai/adapter-utils/duplex-telemetry"; +import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { clampSpanLabel, getActiveStepContext, @@ -206,6 +207,10 @@ export async function resolveEnvironmentExecutionTarget(input: { // enters the sandbox environment. Absent keeps the safe no-op default in the // bridge, so the surface stays inert until the host injects a real recorder. duplexTelemetryRecorder?: DuplexTelemetryRecorder | null; + // The process-owned aggregate byte ledger. The seam stamps it onto the sandbox + // target next to the runner, so the live object stays on the host and never + // enters the sandbox environment. Absent keeps the bridge inert for this seam. + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; }): Promise { if (input.environment.driver === "local") { return { @@ -323,6 +328,10 @@ export async function resolveEnvironmentExecutionTarget(input: { // binds it to the fixed observability surface. Absent keeps the no-op // default, so the surface stays inert on a run with no injected recorder. duplexTelemetryRecorder: input.duplexTelemetryRecorder ?? null, + // Attach the process-owned aggregate byte ledger next to the runner. The + // bridge passes it to the broker, the decoder, and the response-body reader. + // Absent keeps the bridge inert for this seam. + duplexAggregateByteLedger: input.duplexAggregateByteLedger ?? null, ...(effectiveCapabilities ? { effectiveCapabilities: Object.freeze({ ...effectiveCapabilities }) } : {}), environmentId: input.environment.id ?? null, leaseId: input.leaseId ?? null, diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 3a76dcfec13..a6db690eeea 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -43,6 +43,7 @@ import { type AdapterWorkspaceRealization, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexTelemetryRecorder } from "@paperclipai/adapter-utils/duplex-telemetry"; +import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { buildWorkspaceRealizationRequest } from "./workspace-realization.js"; import { executionWorkspaceService } from "./execution-workspaces.js"; import { logActivity } from "./activity-log.js"; @@ -154,6 +155,14 @@ export function environmentRunOrchestrator( options: { pluginWorkerManager?: PluginWorkerManager; environmentRuntime?: EnvironmentRuntimeService; + /** + * The process-owned aggregate byte ledger for the sandbox duplex channel. + * The server root creates one ledger per host process and injects the same + * object here. The orchestrator stamps it onto the sandbox execution target, + * so one shared gauge bounds the aggregate retained bytes across all live + * duplex routes. Absent keeps the bridge inert for this seam. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } = {}, ) { const environmentsSvc = environmentService(db); @@ -523,6 +532,7 @@ export function environmentRunOrchestrator( lease, environmentRuntime, duplexTelemetryRecorder: input.duplexTelemetryRecorder ?? null, + duplexAggregateByteLedger: options.duplexAggregateByteLedger ?? null, }); const realizationMode = workspaceRealization.mode === "in_place" ? "in_place" : "copy"; const authoritativeRoot = diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 611f310fbb1..4a665fc0d15 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -72,6 +72,7 @@ import { import { conflict, HttpError, notFound } from "../errors.js"; import { getStartupTraceContext, getStartupTracer } from "../instrumentation.js"; import { createHostDuplexTelemetryRecorder } from "./duplex-telemetry-recorder.js"; +import type { DuplexAggregateByteLedger } from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { incrementToolRuntimeMetricCounter } from "./tool-runtime-metrics.js"; import { logger } from "../middleware/logger.js"; import { @@ -6692,6 +6693,14 @@ export interface HeartbeatServiceOptions { pluginWorkerManager?: PluginWorkerManager; environmentRuntime?: HeartbeatEnvironmentRuntime; runtimeEnv?: Record; + /** + * The process-owned aggregate byte ledger for the sandbox duplex channel. The + * server root creates one ledger per host process and injects the same object + * here. The heartbeat threads it into the environment run orchestrator, which + * stamps it onto the sandbox execution target. Absent keeps the bridge inert + * for this seam. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } type WorkspaceReadyCommentWriter = { @@ -6815,6 +6824,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const envOrchestrator = environmentRunOrchestrator(db, { pluginWorkerManager: options.pluginWorkerManager, environmentRuntime, + duplexAggregateByteLedger: options.duplexAggregateByteLedger, }); const workspaceOperationsSvc = workspaceOperationService(db); const liveRunExecutions = { diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index 77ac1ff63a8..b755e8388cb 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -57,10 +57,31 @@ import type { InitializeParams, } from "@paperclipai/plugin-sdk"; import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + type DuplexAggregateByteLedger, + type DuplexAggregateTokenOwner, + type ReservationToken, +} from "@paperclipai/adapter-utils/duplex-aggregate-byte-ledger"; import { CLAUDE_SETUP_TOKEN_COMMAND } from "@paperclipai/adapter-claude-local/server"; import { logger } from "../middleware/logger.js"; import { traceparentFromContextToken } from "../instrumentation.js"; +/** + * The host raises this error when the child-stdin transport reservation for a + * duplex write fails against the aggregate byte ledger. The host does not write + * the frame. The write path throws it, `callInternal` rejects the RPC with it + * unwrapped, and the duplex write caller ends the route fail-closed with the + * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED} marker. Only a duplex write + * meters the transport, so this error never reaches a non-duplex control message. + */ +class DuplexAggregateBytesExceededError extends Error { + constructor() { + super(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + this.name = "DuplexAggregateBytesExceededError"; + } +} + // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- @@ -397,6 +418,14 @@ export interface WorkerStartOptions { * constructs a handle this way). */ duplexRouteSlots?: DuplexRouteSlotController | null; + /** + * The process-owned aggregate byte ledger for the duplex channel. The manager + * injects one shared instance into every worker handle, so one gauge bounds the + * aggregate retained bytes across every route in the process. When it is absent, + * the worker retains duplex bytes unbounded (a unit test constructs a handle + * this way). The manager never makes a fresh per-handle default. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; /** * Companies this worker may act on from proactive (no-invocation) worker→host * calls — the plugin's configured companies. Seeded onto the handle at @@ -966,11 +995,42 @@ export function createPluginWorkerHandle( // JSON-RPC message sending // ----------------------------------------------------------------------- - function sendMessage(message: unknown): void { + function sendMessage(message: unknown, meterDuplexWrite = false): void { if (!childProcess?.stdin?.writable) { throw new Error(`Worker process for plugin "${pluginId}" is not writable`); } const serialized = serializeMessage(message as any); + const ledger = duplexAggregateByteLedger; + if (meterDuplexWrite && ledger) { + // Charge the child-stdin transport buffer for a duplex write. The host writes + // the serialized frame to the child stdin without backpressure. When the child + // stops reading its stdin, the frame stays in the host stdin write buffer. + // Reserve the exact serialized-frame byte count. That count includes the JSON + // escaping and the newline framing, so the ledger covers the retained transport + // bytes, not only the raw payload. The RPC separately holds the raw payload + // under a `pending_write` token, so the two tokens cover the peak of both + // retentions at the same time. + const bytes = Buffer.byteLength(serialized); + const token = ledger.reserve("stdin_write", bytes); + if (!token) { + // The reservation would pass the aggregate ceiling. Fail closed before the + // enqueue: do not write the frame. The duplex write caller ends the route. + throw new DuplexAggregateBytesExceededError(); + } + // Hold the token until the stream flushes the chunk. The write callback fires + // when the stream hands the chunk to the operating system, so the bytes then + // leave the host stream buffer. The RPC settle and the RPC timeout never + // release this token. Only the flush, the stream error, the stream close, or + // the worker exit releases it. Release through the outstanding-token set, so a + // later stream-error or worker-exit sweep never double-releases the same token. + pendingStdinWriteTokens.add(token); + childProcess.stdin.write(serialized, () => { + if (pendingStdinWriteTokens.delete(token)) { + ledger.release(token); + } + }); + return; + } childProcess.stdin.write(serialized); } @@ -1550,12 +1610,37 @@ export function createPluginWorkerHandle( // 6. total data bytes — the cumulative inbound bytes over the whole life, // counted before and after a data listener attaches; // 7. route lifetime — the milliseconds from the open to the terminal end. + + // One buffered data chunk retained for a late listener drain. It carries the raw + // chunk string and the aggregate byte token that reserved its raw bytes. The + // token is `null` when no ledger is injected. + interface BufferedDuplexChunk { + chunk: string; + token: ReservationToken | null; + } + // One pre-bind data event, normalized to the narrow duplex-event schema. The host + // retains only these bounded scalar fields plus the reservation token, never the + // original arbitrary notification graph. The bind re-resolves the pair from + // `workerSessionId`, so a frame whose pair does not match the bound pair still + // fails closed. + interface HeldDuplexEvent { + workerSessionId: string; + chunk: string; + token: ReservationToken | null; + } + // One pre-bind exit event, normalized to the narrow duplex-event schema. + interface HeldDuplexExitEvent { + workerSessionId: string; + exitCode: number | null; + token: ReservationToken | null; + } + interface DuplexChannelRoute { hostRouteId: string; state: RouteState; workerSessionId: string | null; listener: ((chunk: string) => void) | null; - buffered: string[]; + buffered: BufferedDuplexChunk[]; bufferedChars: number; pendingRequests: number; protocolErrors: number; @@ -1563,18 +1648,23 @@ export function createPluginWorkerHandle( lifetimeTimer: ReturnType | null; terminalized: boolean; settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void; - // The data frames that arrived before the bind, held in order. The bind - // replays them through the exact-pair routing, so an early frame is never - // lost and a frame whose pair does not match the bound pair still fails - // closed. The hold ceiling is `maxDuplexChannelPreBindHoldFrames`, one frame - // above the buffered bound, so the replay's buffered-bound check ends the - // route, not the hold. - preBind: JsonRpcNotification[]; - // The single exit frame that arrived before the bind. An exit never consumes - // a data hold slot, so a worker that batches an exit among enough data frames - // to fill the hold cannot crowd out a data frame. The bind replays the held - // data frames first, then this exit last. - preBindExit: JsonRpcNotification | null; + // The bounded raw data events that arrived before the bind, held in order. The + // bind replays them through the exact-pair routing, so an early frame is never + // lost and a frame whose pair does not match the bound pair still fails closed. + // The hold ceiling is `maxDuplexChannelPreBindHoldFrames`, one frame above the + // buffered bound, so the replay's buffered-bound check ends the route, not the + // hold. Each held event carries the aggregate byte token that reserved its raw + // bytes; the host never retains the original arbitrary notification graph. + preBind: HeldDuplexEvent[]; + // The single bounded exit event that arrived before the bind. An exit never + // consumes a data hold slot, so a worker that batches an exit among enough data + // frames to fill the hold cannot crowd out a data frame. The bind replays the + // held data events first, then this exit last. + preBindExit: HeldDuplexExitEvent | null; + // The authoritative registry of every aggregate byte token this route still + // holds, across the pre-bind, buffered, and terminal-buffered representations. + // The byte cleanup releases every token here exactly once. + retainedTokens: Set; } // The live duplex routes on this worker, keyed by the exact // `{ hostRouteId, workerSessionId }` pair. The host binds one pair once, at @@ -1600,6 +1690,89 @@ export function createPluginWorkerHandle( // time per route, so a double terminalize never releases two slots. const duplexRouteSlotHolders = new Set(); + // The process-owned aggregate byte ledger. The manager injects it, so one gauge + // bounds the aggregate retained bytes across every route in the process. When it + // is absent, the worker retains duplex bytes unbounded (a unit test constructs it + // this way). + const duplexAggregateByteLedger = options.duplexAggregateByteLedger ?? null; + // The outstanding child-stdin transport tokens for duplex writes. Each token + // covers one serialized frame the host stdin write buffer still retains. The + // write callback releases a token on the flush; a stream error, a stream close, + // or a worker exit releases every remaining token, because each of those + // discards the stdin write buffer. The set is the release guard, so a token + // releases one time across the two paths. + const pendingStdinWriteTokens = new Set(); + // Release every outstanding child-stdin transport token. A stream error, a + // stream close, or a worker exit calls this, because each discards the stdin + // write buffer. The `delete` guard drops each token one time, so a later flush + // callback or a second sweep releases nothing again. + function releaseAllPendingStdinWriteTokens(): void { + if (!duplexAggregateByteLedger) return; + for (const token of pendingStdinWriteTokens) { + duplexAggregateByteLedger.release(token); + } + pendingStdinWriteTokens.clear(); + } + // The terminalized routes that still hold buffered bytes for a late listener + // drain. A terminalized route leaves the opening and live maps, so this registry + // keeps the worker-exit sweep able to release its still-charged buffered tokens. + const terminalDuplexRoutes = new Set(); + + // Reserve `bytes` for one route retention against the aggregate ledger. Return + // the held token, or `null` when the reservation would pass the ceiling. When no + // ledger is present, return `"no-ledger"`, so the caller admits the retention + // with no token. The helper adds a real token to `route.retainedTokens`. + function reserveRouteBytes( + route: DuplexChannelRoute, + owner: DuplexAggregateTokenOwner, + bytes: number, + ): ReservationToken | "no-ledger" | null { + if (!duplexAggregateByteLedger) return "no-ledger"; + const token = duplexAggregateByteLedger.reserve(owner, bytes); + if (!token) return null; + route.retainedTokens.add(token); + return token; + } + + // Transfer a held route token to a new owner label. The token identity and the + // reserved bytes stay the same, so no admission gap opens between two + // representations of the same retained bytes. + function transferRouteToken( + token: ReservationToken | null, + owner: DuplexAggregateTokenOwner, + ): void { + if (!token) return; + duplexAggregateByteLedger?.transfer(token, owner); + } + + // Release one held route token and drop it from the route registry, in one + // synchronous step. A `null` token (no ledger) releases nothing. + function releaseRouteToken(route: DuplexChannelRoute, token: ReservationToken | null): void { + if (!token) return; + route.retainedTokens.delete(token); + duplexAggregateByteLedger?.release(token); + } + + // Release every token a route still holds and clear its retained representations, + // exactly once. A later call finds an empty registry and releases nothing, so the + // helper is idempotent. Every terminal-discard, open-failure, bind-replay + // failure, close-acknowledgement failure, worker-exit, and shutdown path calls + // it. Terminal map deletion is never treated as proof that retained bytes are + // gone; this registry is authoritative for byte cleanup. + function discardRouteRetained(route: DuplexChannelRoute): void { + if (duplexAggregateByteLedger) { + for (const token of route.retainedTokens) { + duplexAggregateByteLedger.release(token); + } + } + route.retainedTokens.clear(); + route.preBind = []; + route.preBindExit = null; + route.buffered = []; + route.bufferedChars = 0; + terminalDuplexRoutes.delete(route); + } + // Try to reserve one aggregate route slot for a route. Return true when the // route holds a slot after the call. When no controller is present, the route // always holds a slot. @@ -1697,15 +1870,38 @@ export function createPluginWorkerHandle( route.terminalized = true; route.state = "closed"; route.listener = null; - // Keep the buffered chunks the host accepted before the route ended, so a - // listener that attaches after the end still drains them. A frame can end the - // route during the pre-bind replay, before a listener attaches, and the - // chunks the host accepted before that frame are valid data the listener must - // still receive. The buffered bytes stay bounded by the pre-bind buffered - // bound, and `onData` clears them once it drains them. Drop the held pre-bind - // frames, which never bound to a listener. + // Stop admission first. Then move the byte cleanup. Keep the buffered chunks the + // host accepted before the route ended, so a listener that attaches after the + // end still drains them. A frame can end the route during the pre-bind replay, + // before a listener attaches, and the chunks the host accepted before that frame + // are valid data the listener must still receive. The buffered bytes stay + // bounded by the pre-bind buffered bound, and `onData` clears and releases them + // once it drains them. + // + // The buffered records keep their tokens; move each token to the terminal owner + // label and register the route in the terminal registry, so the worker-exit + // sweep can release the still-charged bytes later. Do not release a buffered + // token at map deletion; terminal map deletion is never proof the bytes are + // gone. Release every other token the route still holds (the pre-bind events, + // the held exit, and any stranded token), because they never reach a listener. + const bufferedTokens = new Set(); + for (const record of route.buffered) { + if (record.token) { + bufferedTokens.add(record.token); + transferRouteToken(record.token, "terminal_buffered"); + } + } + for (const token of [...route.retainedTokens]) { + if (!bufferedTokens.has(token)) { + route.retainedTokens.delete(token); + duplexAggregateByteLedger?.release(token); + } + } route.preBind = []; route.preBindExit = null; + if (route.buffered.length > 0) { + terminalDuplexRoutes.add(route); + } clearDuplexChannelLifetimeTimer(route); // Remove the live binding and install the tombstone in one synchronous step, // before the worker close and before any reuse. A reserved route that never @@ -1787,14 +1983,26 @@ export function createPluginWorkerHandle( return "violation"; } - function routeDuplexChannelData(notification: JsonRpcNotification): void { + // Route one duplex channel data frame. On a live route with a listener, deliver + // the chunk transiently. On a live route with no listener, buffer the chunk and + // charge its raw bytes against the aggregate ledger. The `carried` argument marks + // a bind-replay frame: it carries the pre-bind token the buffered record must + // reuse, so the replay transfers the token to the buffered representation with no + // decrement and no re-reserve. A live frame passes no `carried`, so it reserves a + // fresh buffered token. A `carried.token` of `null` means no ledger is present. + function routeDuplexChannelData( + notification: JsonRpcNotification, + carried?: { token: ReservationToken | null }, + ): void { const params = isRecord(notification.params) ? notification.params : {}; const hostRouteId = readNonEmptyString(params.hostRouteId); const workerSessionId = readNonEmptyString(params.workerSessionId); const resolved = resolveDuplexRouteByPair(hostRouteId, workerSessionId); if (resolved === "tombstoned") { // A late frame for a closed pair. It reaches no listener and changes no - // state. Never log the raw frame content. + // state. A bind replay never lands here, because the replayed route is live; + // a carried token stays on its route and the terminal cleanup releases it. + // Never log the raw frame content. return; } if (resolved === "violation") { @@ -1804,8 +2012,8 @@ export function createPluginWorkerHandle( return; } if (resolved === "opening") { - // The frame arrived before the bind. Hold it; the bind replays it through the - // exact-pair routing. + // The frame arrived before the bind. A replay never lands here, because the + // route is live during replay. Hold the live frame; the bind replays it. bufferPreBindDuplexFrame(hostRouteId, notification); return; } @@ -1813,13 +2021,15 @@ export function createPluginWorkerHandle( const chunk = params.chunk; if (typeof chunk !== "string" || chunk.length === 0) { // The exact pair matches, but the chunk is malformed. Count one per-route - // protocol error. + // protocol error. Release a carried replay token first. + releaseRouteToken(route, carried?.token ?? null); recordDuplexChannelProtocolError(route); return; } if (chunk.length > maxDuplexChannelChunkChars) { // One inbound chunk is larger than the per-chunk limit. End the route at // once. Do not count the chunk as a protocol error. + releaseRouteToken(route, carried?.token ?? null); void terminalizeDuplexChannelRoute(route); return; } @@ -1828,12 +2038,16 @@ export function createPluginWorkerHandle( // bound listener cannot receive data past the cap. const chunkBytes = Buffer.byteLength(chunk); if (route.totalDataBytes + chunkBytes > maxDuplexChannelTotalDataBytes) { + releaseRouteToken(route, carried?.token ?? null); void terminalizeDuplexChannelRoute(route); return; } route.totalDataBytes += chunkBytes; if (route.listener) { + // A listener is attached. Deliver the chunk transiently and release a carried + // replay token after the synchronous delivery boundary. deliverDuplexChannelChunk(route.listener, chunk); + releaseRouteToken(route, carried?.token ?? null); return; } // No listener attached yet. Buffer the frame under the pre-bind bounds. End @@ -1842,10 +2056,28 @@ export function createPluginWorkerHandle( route.buffered.length + 1 > maxDuplexChannelPreBindFrames || route.bufferedChars + chunk.length > maxDuplexChannelPreBindChars ) { + releaseRouteToken(route, carried?.token ?? null); void terminalizeDuplexChannelRoute(route); return; } - route.buffered.push(chunk); + let token: ReservationToken | null; + if (carried) { + // The bind replay transfers the pre-bind token to the buffered + // representation. No decrement and no re-reserve, so no admission gap opens. + token = carried.token; + transferRouteToken(token, "buffered_chunk"); + } else { + // A live frame reserves a fresh buffered token for its exact raw bytes. + const reserved = reserveRouteBytes(route, "buffered_chunk", chunkBytes); + if (reserved === null) { + // The aggregate ceiling rejected the reservation. Retain nothing and fail + // closed with the fixed marker. + void terminalizeDuplexChannelRoute(route); + return; + } + token = reserved === "no-ledger" ? null : reserved; + } + route.buffered.push({ chunk, token }); route.bufferedChars += chunk.length; } @@ -1900,15 +2132,58 @@ export function createPluginWorkerHandle( if (!hostRouteId) return; const route = openingDuplexRoutes.get(hostRouteId); if (!route) return; + const params = isRecord(notification.params) ? notification.params : {}; + const workerSessionId = readNonEmptyString(params.workerSessionId); + if (!workerSessionId) { + // A malformed pair. Count one per-route protocol error and hold nothing. + recordDuplexChannelProtocolError(route); + return; + } if (notification.method === DUPLEX_CHANNEL_EXIT_NOTIFICATION) { - route.preBindExit = notification; + // Normalize the exit to the narrow duplex-event schema. An exit retains only + // a bounded scalar, so reserve a zero-byte token that carries the record + // through the one cleanup path. Release any earlier held exit token first, so + // a replaced exit never leaks its reservation. + const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; + if (route.preBindExit) releaseRouteToken(route, route.preBindExit.token); + const reserved = reserveRouteBytes(route, "pre_bind_event", 0); + if (reserved === null) { + // The aggregate ceiling rejected the reservation. Fail closed. + void terminalizeDuplexChannelRoute(route); + return; + } + route.preBindExit = { + workerSessionId, + exitCode, + token: reserved === "no-ledger" ? null : reserved, + }; + return; + } + // A data event. Validate and normalize it to the narrow duplex-event schema + // before any retention. + const chunk = params.chunk; + if (typeof chunk !== "string" || chunk.length === 0) { + recordDuplexChannelProtocolError(route); return; } if (route.preBind.length >= maxDuplexChannelPreBindHoldFrames) { recordDuplexChannelProtocolError(route); return; } - route.preBind.push(notification); + // Reserve the exact retained raw byte count before the host holds the event. + const reserved = reserveRouteBytes(route, "pre_bind_event", Buffer.byteLength(chunk)); + if (reserved === null) { + // The aggregate ceiling rejected the reservation. The caller retains nothing + // and the route fails closed with the fixed marker. + log.warn({ pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED }, "duplex pre-bind hold rejected"); + void terminalizeDuplexChannelRoute(route); + return; + } + route.preBind.push({ + workerSessionId, + chunk, + token: reserved === "no-ledger" ? null : reserved, + }); } // Replay the frames a route held before it bound. The route is live now, so the @@ -1921,14 +2196,47 @@ export function createPluginWorkerHandle( function replayPreBindDuplexFrames(route: DuplexChannelRoute): void { const held = route.preBind; route.preBind = []; - for (const notification of held) { - if (route.terminalized) break; - routeDuplexChannelData(notification); + for (const event of held) { + if (route.terminalized) { + // The route ended mid-replay. Release the remaining held tokens, so the + // unreplayed events retain nothing. + releaseRouteToken(route, event.token); + continue; + } + // Reconstruct a transient data notification for the exact-pair routing. The + // host holds only the bounded event, so it builds this notification for the + // routing step and discards it at once. The carried token moves to the + // buffered representation without a decrement or a re-reserve. + routeDuplexChannelData( + { + jsonrpc: "2.0", + method: DUPLEX_CHANNEL_DATA_NOTIFICATION, + params: { + hostRouteId: route.hostRouteId, + workerSessionId: event.workerSessionId, + chunk: event.chunk, + }, + }, + { token: event.token }, + ); } const heldExit = route.preBindExit; route.preBindExit = null; - if (heldExit && !route.terminalized) { - routeDuplexChannelExit(heldExit); + if (heldExit) { + // The exit retains only a bounded scalar. Release its placeholder token, then + // resolve the wait through the exact-pair routing when the route still lives. + releaseRouteToken(route, heldExit.token); + if (!route.terminalized) { + routeDuplexChannelExit({ + jsonrpc: "2.0", + method: DUPLEX_CHANNEL_EXIT_NOTIFICATION, + params: { + hostRouteId: route.hostRouteId, + workerSessionId: heldExit.workerSessionId, + exitCode: heldExit.exitCode, + }, + }); + } } } @@ -1939,23 +2247,34 @@ export function createPluginWorkerHandle( // id never returns, so no closed pair can revive on a restart. The pending // channel calls reject through `rejectAllPending`. function closeDuplexChannelRouteOnWorkerExit(): void { - const routes = [...openingDuplexRoutes.values(), ...liveDuplexRoutes.values()]; + // Enumerate the opening index, the live index, and the terminal registry. A + // terminalized route left the opening and live maps but may still hold buffered + // bytes for a late listener; the worker is gone, so the host releases them now. + const routes = [ + ...openingDuplexRoutes.values(), + ...liveDuplexRoutes.values(), + ...terminalDuplexRoutes, + ]; openingDuplexRoutes.clear(); liveDuplexRoutes.clear(); duplexPairTombstones.clear(); for (const route of routes) { - if (route.terminalized) continue; - route.terminalized = true; - route.state = "closed"; - route.listener = null; - route.buffered = []; - route.bufferedChars = 0; - route.preBind = []; - route.preBindExit = null; - clearDuplexChannelLifetimeTimer(route); - releaseDuplexRouteSlot(route); - settleRouteWait(route, { exitCode: null }); + if (!route.terminalized) { + route.terminalized = true; + route.state = "closed"; + route.listener = null; + route.bufferedChars = 0; + clearDuplexChannelLifetimeTimer(route); + releaseDuplexRouteSlot(route); + settleRouteWait(route, { exitCode: null }); + } + // Release every token the route still holds and clear its retained + // representations, exactly once. A route already drained by a late listener + // holds no token, so this is harmless and leaves the ledger at zero. + discardRouteRetained(route); } + // Clear the terminal registry. Every route in it was just discarded above. + terminalDuplexRoutes.clear(); } // Open one live generic duplex channel route. Reserve the route before the open @@ -1988,6 +2307,7 @@ export function createPluginWorkerHandle( settleWait, preBind: [], preBindExit: null, + retainedTokens: new Set(), }; // Reserve one aggregate route slot before any work. When the process-scoped // ceiling is full, reject with the fixed route-busy error and open nothing, so @@ -2074,11 +2394,61 @@ export function createPluginWorkerHandle( void terminalizeDuplexChannelRoute(route); return; } + // Reserve the exact UTF-8 byte count of a host→worker write against the + // aggregate ledger before `callInternal` retains the payload. A pending write + // RPC holds `params.data` until it settles, so this reservation bounds the + // aggregate host→worker pending-write bytes across every route. Compute the + // byte count with `Buffer.byteLength`, not `data.length`, because one + // character can encode as several UTF-8 bytes. A stop request carries no + // payload, so it reserves nothing. When no ledger is present, admit the write + // with no token (a unit test constructs the handle this way). + let pendingWriteToken: ReservationToken | null = null; + if (method === "duplexChannelWrite" && duplexAggregateByteLedger) { + const data = (params as HostToWorkerMethods["duplexChannelWrite"][0]).data; + const bytes = Buffer.byteLength(data, "utf8"); + pendingWriteToken = duplexAggregateByteLedger.reserve("pending_write", bytes); + if (!pendingWriteToken) { + // The reservation would pass the aggregate ceiling. Retain nothing, do + // not enqueue the RPC, and end the route fail-closed with the aggregate + // marker, not the route-busy marker. + log.warn( + { pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED }, + "duplex pending write reservation rejected", + ); + void terminalizeDuplexChannelRoute(route); + return; + } + } route.pendingRequests += 1; - void callInternal(method, params, duplexChannelOpenTimeoutMs) - .catch(() => {}) + // Meter the child-stdin transport buffer only for a duplex write. A stop + // request carries a tiny fixed frame that the host never lets grow, so it + // does not meter or reject. The write path reserves the serialized frame in + // `sendMessage` right before the stdin write. + const meterDuplexWrite = method === "duplexChannelWrite"; + void callInternal(method, params, duplexChannelOpenTimeoutMs, undefined, meterDuplexWrite) + .catch((err: unknown) => { + if (err instanceof DuplexAggregateBytesExceededError) { + // The transport reservation failed. The host did not write the frame. + // End the route fail-closed with the aggregate marker, not the + // route-busy marker. + log.warn( + { pluginId, reason: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED }, + "duplex stdin write reservation rejected", + ); + void terminalizeDuplexChannelRoute(route); + } + }) .finally(() => { route.pendingRequests -= 1; + // Release the pending-write token one time, after the RPC settles on any + // path: success, error, timeout, worker exit, or shutdown. The token is + // not in `route.retainedTokens`, so route terminalization never releases + // it; only this settlement releases it. The separate `stdin_write` token + // covers the serialized frame and releases on the stream flush, the + // stream error, the stream close, or the worker exit, never here. + if (pendingWriteToken) { + duplexAggregateByteLedger?.release(pendingWriteToken); + } }); }; @@ -2089,7 +2459,16 @@ export function createPluginWorkerHandle( const pending = route.buffered; route.buffered = []; route.bufferedChars = 0; - for (const chunk of pending) deliverDuplexChannelChunk(listener, chunk); + for (const record of pending) { + // Deliver each buffered record, drop its retained buffer, and release + // its exact token before the drain proceeds to the next record. A + // terminal route drains through this same code. + deliverDuplexChannelChunk(listener, record.chunk); + releaseRouteToken(route, record.token); + } + // The buffered records are gone, so the route no longer holds terminal + // bytes for a late listener. + terminalDuplexRoutes.delete(route); } }, write(data: string): void { @@ -2392,6 +2771,18 @@ export function createPluginWorkerHandle( readline.on("line", handleLine); } + // Release the outstanding child-stdin transport tokens when the stdin stream + // errors or closes, because each discards the stdin write buffer. The `error` + // listener also stops an unhandled EPIPE from a child that closed its stdin. + if (child.stdin) { + child.stdin.on("error", () => { + releaseAllPendingStdinWriteTokens(); + }); + child.stdin.on("close", () => { + releaseAllPendingStdinWriteTokens(); + }); + } + // Capture stderr for logging if (child.stderr) { stderrReadline = createInterface({ input: child.stderr }); @@ -2442,6 +2833,12 @@ export function createPluginWorkerHandle( childProcess = null; startedAt = null; + // The worker exit discards the child-stdin write buffer, so release every + // outstanding transport token. The RPC rejections below never release these + // tokens; only this sweep, a stream flush, a stream error, or a stream close + // releases them. + releaseAllPendingStdinWriteTokens(); + // Reject all pending requests rejectAllPending( new Error(formatWorkerFailureMessage( @@ -2777,6 +3174,7 @@ export function createPluginWorkerHandle( params: HostToWorkerMethods[M][0], timeoutMs?: number, executeLogSink?: ExecuteLogSink, + meterDuplexWrite = false, ): Promise { const rpcPromise = new Promise((resolve, reject) => { if (!childProcess?.stdin?.writable) { @@ -2850,19 +3248,26 @@ export function createPluginWorkerHandle( ...createRequest(method, params, id), ...(invocation ? { paperclipInvocation: invocation } : {}), }; - sendMessage(request); + sendMessage(request, meterDuplexWrite); } catch (err) { clearTimeout(timer); pendingRequests.delete(id); clearInvocation(invocation); clearExecuteRoute(invocation?.id); - reject( - new Error( - `Failed to send "${method}" to worker: ${ - err instanceof Error ? err.message : String(err) - }`, - ), - ); + if (err instanceof DuplexAggregateBytesExceededError) { + // The transport reservation failed before the write. Reject with the + // typed error unwrapped, so the duplex write caller ends the route + // fail-closed with the aggregate marker. + reject(err); + } else { + reject( + new Error( + `Failed to send "${method}" to worker: ${ + err instanceof Error ? err.message : String(err) + }`, + ), + ); + } } }); @@ -3035,6 +3440,14 @@ export interface PluginWorkerManagerOptions { * stays upstream admission only. */ maxConcurrentDuplexRoutes?: number | null; + /** + * The process-owned aggregate byte ledger. The process root creates one ledger + * from validated configuration and passes it here. The manager injects the same + * object into every worker handle, so one gauge bounds the aggregate retained + * bytes across every route in the process. When it is absent, the worker retains + * duplex bytes unbounded. The manager never makes a fresh default ledger. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } /** @@ -3106,6 +3519,11 @@ export function createPluginWorkerManager( const duplexRouteSlots = createDuplexRouteSlotController( managerOptions?.maxConcurrentDuplexRoutes, ); + // The one shared, process-owned aggregate byte ledger. The manager injects the + // same object into every worker handle, so one gauge bounds the aggregate + // retained bytes across every route in the process. It is `null` when the + // process root injected no ledger. + const duplexAggregateByteLedger = managerOptions?.duplexAggregateByteLedger ?? null; return { async startWorker( @@ -3127,9 +3545,11 @@ export function createPluginWorkerManager( } const handle = createPluginWorkerHandle(pluginId, { - // Inject the shared process-scoped route-slot controller, unless the caller - // already supplied one (a test may inject its own). + // Inject the shared process-scoped route-slot controller and the shared + // process-owned aggregate byte ledger, unless the caller already supplied + // its own (a test may inject its own). duplexRouteSlots, + duplexAggregateByteLedger, ...options, }); workers.set(pluginId, handle); From 8db826d18a43a48ac969256bf5311da96bb1f1eb Mon Sep 17 00:00:00 2001 From: zach-hermes Date: Sun, 23 Aug 2026 05:50:38 -1000 Subject: [PATCH 4/6] fix(issues): cycle-aware issue_blockers_resolved after terminal reset (#11979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Agents resume blocked work through the `issue_blockers_resolved` wake when every durable blocker is `done` > - That wake is level-triggered: one ready state produces one wake, shared by the issue update route, workspace-finalize backstop, and periodic liveness backstop > - The ready-state key hashed only the dependent id and blocker set, so it ignored a later reset from a terminal status back into `blocked` > - After that reset, completing the same blockers found the previous cycle's completed wake and suppressed the new continuation > - This pull request folds the dependent's `blockedTransitionAt` into the ready-state key, with compatibility for old no-cycle keys > - The benefit is that a reset blocked issue receives exactly one new wake without watchdog status repair or a change to blocker edges ## Linked Issues or Issue Description Refs: https://github.com/paperclipai/paperclip/issues/5985 Refs: https://github.com/paperclipai/paperclip/issues/6555 Related: https://github.com/paperclipai/paperclip/pull/8009 Related: https://github.com/paperclipai/paperclip/pull/11570 This change does not auto-flip `blocked` to `todo`. The wake is the continuation. It also does not treat cancelled blockers as resolved. **What happened?** A blocked assigned issue that was previously `done` or `cancelled`, then reset to `blocked` on the same blocker set, did not receive `issue_blockers_resolved` when those blockers later returned to `done`. A completed wake from the previous cycle reused the same level-triggered state key and suppressed the new wake. Route-time emit, workspace-finalize backstop, and periodic liveness backstop all used that helper. **Expected behavior** When every durable blocker is `done`, a currently `blocked` assigned issue must receive exactly one valid `issue_blockers_resolved` continuation for the current blocked cycle. A completed wake from an earlier cycle must not suppress it. Watchdog `blocked` → `todo` repair must not be required. **Steps to reproduce** 1. Assign issue B, block it on issue A, mark A `done`, and let B receive `issue_blockers_resolved`. 2. Mark B `done`. 3. Reset A to `todo` and reset B from `done` to `blocked` on the same A id. This refreshes `blockedTransitionAt`. 4. Mark A `done` again. 5. Observe that B stays `blocked` with no new `issue_blockers_resolved` wake. **Paperclip version or commit** `master` at `cc42a67e7e9e8eb183097afc8ff4ebfa694fb3e0` **Deployment mode** Self-hosted server ## What Changed - Extend `buildIssueBlockersResolvedWakeStateKey` so the digest includes the dependent's `blockedTransitionAt` as UTC ISO-8601, or `none` - Thread `blockedTransitionAt` through `listWakeableBlockedDependents`, both route emit sites, and both backstop candidate selects - Keep compatibility: new cycle-aware keys suppress in idempotent statuses; old no-cycle state keys suppress when in-flight, or when completed and `requestedAt >= blockedTransitionAt` (or the cycle is null); legacy per-edge keys stay in-flight-only - Do not rewrite `blockedByIssueIds`, auto-flip `blocked` → `todo`, or delete historical wake rows - Add helper, route, restore, chained dependent, and backstop tests for the reset cycle ## Verification ``` pnpm --filter @paperclipai/server exec vitest run \ src/__tests__/issue-dependency-wakeups-routes.test.ts \ src/__tests__/heartbeat-issue-liveness-escalation.test.ts \ src/services/issue-dependency-wakeups.ts \ src/services/issue-dependency-wakeups.test.ts ``` Local result: all named tests passed (helper 9, routes 8, liveness 26). ## Risks - Deploy overlap: in-flight and same-cycle completed wakes still exist under the old no-cycle key. The lookup keeps those as suppressors so this change does not enqueue a duplicate in the current cycle. - A completed old-key wake from before the current `blockedTransitionAt` no longer suppresses. That is the intended fix. - No schema migration. Rollback is revert of this PR. - This does not change cancelled-blocker semantics or watchdog `blocked` → `todo` repair. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Provider: xAI - Model: Grok 4.6 - Tool use and code execution: yes - Human-authored: no ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes: #` / `Refs: #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- ...eartbeat-issue-liveness-escalation.test.ts | 100 ++++++- .../issue-dependency-wakeups-routes.test.ts | 281 +++++++++++++++++- server/src/routes/issues.ts | 9 + .../services/issue-dependency-wakeups.test.ts | 231 ++++++++++++++ .../src/services/issue-dependency-wakeups.ts | 202 +++++++++---- server/src/services/issues.ts | 2 + server/src/services/recovery/service.ts | 4 + 7 files changed, 776 insertions(+), 53 deletions(-) create mode 100644 server/src/services/issue-dependency-wakeups.test.ts diff --git a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts index 1f6d5b72a86..7619972cd2c 100644 --- a/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts +++ b/server/src/__tests__/heartbeat-issue-liveness-escalation.test.ts @@ -72,7 +72,10 @@ import { instanceSettingsService } from "../services/instance-settings.ts"; import { issueService } from "../services/issues.ts"; import { runningProcesses } from "../adapters/index.ts"; import { DEFAULT_LIVENESS_REESCALATION_COOLDOWN_MS } from "../services/recovery/service.ts"; -import { buildIssueBlockersResolvedWakeStateKey } from "../services/issue-dependency-wakeups.ts"; +import { + buildIssueBlockersResolvedWakeStateKey, + buildIssueBlockersResolvedWakeStateKeyWithoutCycle, +} from "../services/issue-dependency-wakeups.ts"; const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -812,6 +815,101 @@ describeEmbeddedPostgres("heartbeat issue graph liveness escalation", () => { expect(stateKeyWakes).toHaveLength(1); }); + it("heals a blocked dependent after a terminal reset when a previous-cycle old-key wake exists", async () => { + await enableAutoRecovery(); + const { companyId, agentId, blockedIssueId, blockerIssueId } = + await seedResolvedDependencyBackstopFixture({ workspaceState: "none" }); + const previousCycleWakeAt = new Date("2026-07-01T12:00:00.000Z"); + const blockedTransitionAt = new Date("2026-08-01T12:00:00.000Z"); + await db + .update(issues) + .set({ blockedTransitionAt, updatedAt: blockedTransitionAt }) + .where(eq(issues.id, blockedIssueId)); + await db.insert(agentWakeupRequests).values({ + companyId, + agentId, + source: "automation", + triggerDetail: "system", + reason: "issue_blockers_resolved", + payload: { + issueId: blockedIssueId, + resolvedBlockerIssueId: blockerIssueId, + blockerIssueIds: [blockerIssueId], + }, + status: "completed", + finishedAt: previousCycleWakeAt, + requestedAt: previousCycleWakeAt, + idempotencyKey: buildIssueBlockersResolvedWakeStateKeyWithoutCycle({ + dependentIssueId: blockedIssueId, + blockerIssueIds: [blockerIssueId], + }), + }); + + const result = await heartbeatService(db).reconcileIssueGraphLiveness(); + + expect(result.dependencyWakesHealed).toBe(1); + expect(result.dependencyWakeIssueIds).toEqual([blockedIssueId]); + expect(result.dependencyWakeExistingSkipped).toBe(0); + + const cycleKey = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId: blockedIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt, + }); + const healedWake = await db + .select({ status: agentWakeupRequests.status, idempotencyKey: agentWakeupRequests.idempotencyKey }) + .from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.idempotencyKey, cycleKey))) + .then((rows) => rows[0] ?? null); + expect(healedWake).not.toBeNull(); + expect(["queued", "claimed", "completed"]).toContain(healedWake?.status); + + const secondPass = await heartbeatService(db).reconcileIssueGraphLiveness(); + expect(secondPass.dependencyWakesHealed).toBe(0); + + const cycleKeyWakes = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.idempotencyKey, cycleKey))); + expect(cycleKeyWakes).toHaveLength(1); + }); + + it("does not re-heal when a completed old-key wake is from the current blocked cycle", async () => { + await enableAutoRecovery(); + const { companyId, agentId, blockedIssueId, blockerIssueId } = + await seedResolvedDependencyBackstopFixture({ workspaceState: "none" }); + const blockedTransitionAt = new Date("2026-08-01T12:00:00.000Z"); + const sameCycleWakeAt = new Date("2026-08-01T12:00:01.000Z"); + await db + .update(issues) + .set({ blockedTransitionAt, updatedAt: blockedTransitionAt }) + .where(eq(issues.id, blockedIssueId)); + await db.insert(agentWakeupRequests).values({ + companyId, + agentId, + source: "automation", + triggerDetail: "system", + reason: "issue_blockers_resolved", + payload: { + issueId: blockedIssueId, + resolvedBlockerIssueId: blockerIssueId, + blockerIssueIds: [blockerIssueId], + }, + status: "completed", + finishedAt: sameCycleWakeAt, + requestedAt: sameCycleWakeAt, + idempotencyKey: buildIssueBlockersResolvedWakeStateKeyWithoutCycle({ + dependentIssueId: blockedIssueId, + blockerIssueIds: [blockerIssueId], + }), + }); + + const result = await heartbeatService(db).reconcileIssueGraphLiveness(); + + expect(result.dependencyWakesHealed).toBe(0); + expect(result.dependencyWakeExistingSkipped).toBe(1); + }); + it("counts null dependency wake returns as deferred instead of enqueue failures", async () => { await enableAutoRecovery(); const { companyId, agentId } = diff --git a/server/src/__tests__/issue-dependency-wakeups-routes.test.ts b/server/src/__tests__/issue-dependency-wakeups-routes.test.ts index a840a81a4af..16cf34a20ab 100644 --- a/server/src/__tests__/issue-dependency-wakeups-routes.test.ts +++ b/server/src/__tests__/issue-dependency-wakeups-routes.test.ts @@ -1,6 +1,9 @@ import express from "express"; import request from "supertest"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + buildIssueBlockersResolvedWakeStateKey, +} from "../services/issue-dependency-wakeups.ts"; // The first test in this suite imports the large `routes/issues.ts` module // through `vi.importActual` inside `createApp`. `vi.resetModules()` in @@ -11,7 +14,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; // default 5000ms test timeout and produce a flaky "Test timed out in 5000ms" // failure. Give the suite generous headroom, far above the observed cold-start // yet still below the 30s hook timeout. -vi.setConfig({ testTimeout: 15000 }); +vi.setConfig({ testTimeout: 30000 }); const mockWakeup = vi.hoisted(() => vi.fn(async () => undefined)); const mockFindExistingIssueBlockersResolvedWakeForReadyState = vi.hoisted(() => vi.fn(async () => null)); @@ -402,4 +405,280 @@ describe("issue dependency wakeups in issue routes", () => { ); }); }); + + function issueRecord(overrides: Record = {}) { + return { + id: "issue-1", + companyId: "company-1", + identifier: "PAP-100", + title: "Issue", + description: null, + status: "todo", + priority: "medium", + parentId: null, + assigneeAgentId: "agent-1", + assigneeUserId: null, + createdByAgentId: null, + createdByUserId: null, + executionWorkspaceId: null, + blockedTransitionAt: null, + labels: [], + labelIds: [], + ...overrides, + }; + } + + it("wakes a Release-like dependent after a terminal reset using the current blocked cycle", async () => { + const reviewIssueId = "11111111-1111-4111-8111-111111111111"; + const releaseIssueId = "22222222-2222-4222-8222-222222222222"; + const releaseBlockedAt = new Date("2026-08-01T15:00:00.000Z"); + mockIssueService.getById.mockResolvedValue(issueRecord({ + id: reviewIssueId, + identifier: "PAP-REVIEW", + title: "Review", + status: "in_progress", + })); + mockIssueService.update.mockResolvedValue(issueRecord({ + id: reviewIssueId, + identifier: "PAP-REVIEW", + title: "Review", + status: "done", + })); + mockIssueService.listWakeableBlockedDependents.mockResolvedValue([ + { + id: releaseIssueId, + assigneeAgentId: "agent-release", + blockerIssueIds: [reviewIssueId], + blockedTransitionAt: releaseBlockedAt, + }, + ]); + + const res = await request(await createApp()).patch(`/api/issues/${reviewIssueId}`).send({ status: "done" }); + expect(res.status).toBe(200); + await vi.waitFor(() => { + expect(mockFindExistingIssueBlockersResolvedWakeForReadyState).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + companyId: "company-1", + dependentIssueId: releaseIssueId, + blockerIssueIds: [reviewIssueId], + blockedTransitionAt: releaseBlockedAt, + }), + ); + expect(mockWakeup).toHaveBeenCalledTimes(1); + expect(mockWakeup).toHaveBeenCalledWith( + "agent-release", + expect.objectContaining({ + reason: "issue_blockers_resolved", + idempotencyKey: buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId: releaseIssueId, + blockerIssueIds: [reviewIssueId], + blockedTransitionAt: releaseBlockedAt, + }), + payload: expect.objectContaining({ + issueId: releaseIssueId, + resolvedBlockerIssueId: reviewIssueId, + mutation: "blocker_done", + }), + }), + ); + }); + }); + + it("does not enqueue a second wake when the blocker is already done", async () => { + const reviewIssueId = "11111111-1111-4111-8111-111111111111"; + mockIssueService.getById.mockResolvedValue(issueRecord({ + id: reviewIssueId, + identifier: "PAP-REVIEW", + title: "Review", + status: "done", + })); + mockIssueService.update.mockResolvedValue(issueRecord({ + id: reviewIssueId, + identifier: "PAP-REVIEW", + title: "Review", + status: "done", + })); + mockIssueService.listWakeableBlockedDependents.mockResolvedValue([ + { + id: "22222222-2222-4222-8222-222222222222", + assigneeAgentId: "agent-release", + blockerIssueIds: [reviewIssueId], + blockedTransitionAt: new Date("2026-08-01T15:00:00.000Z"), + }, + ]); + + const res = await request(await createApp()).patch(`/api/issues/${reviewIssueId}`).send({ status: "done" }); + expect(res.status).toBe(200); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(mockWakeup).not.toHaveBeenCalled(); + expect(mockIssueService.listWakeableBlockedDependents).not.toHaveBeenCalled(); + }); + + it("wakes a QA-like chain one dependent at a time after each blocker completes", async () => { + const reviewIssueId = "11111111-1111-4111-8111-111111111111"; + const releaseIssueId = "22222222-2222-4222-8222-222222222222"; + const qaIssueId = "33333333-3333-4333-8333-333333333333"; + const releaseBlockedAt = new Date("2026-08-02T10:00:00.000Z"); + const qaBlockedAt = new Date("2026-08-02T10:05:00.000Z"); + const app = await createApp(); + + mockIssueService.getById.mockResolvedValue(issueRecord({ + id: reviewIssueId, + identifier: "PAP-REVIEW", + title: "Review", + status: "in_progress", + })); + mockIssueService.update.mockResolvedValue(issueRecord({ + id: reviewIssueId, + identifier: "PAP-REVIEW", + title: "Review", + status: "done", + })); + mockIssueService.listWakeableBlockedDependents.mockResolvedValue([ + { + id: releaseIssueId, + assigneeAgentId: "agent-release", + blockerIssueIds: [reviewIssueId], + blockedTransitionAt: releaseBlockedAt, + }, + ]); + + expect((await request(app).patch(`/api/issues/${reviewIssueId}`).send({ status: "done" })).status).toBe(200); + await vi.waitFor(() => { + expect(mockWakeup).toHaveBeenCalledTimes(1); + }); + expect(mockWakeup).toHaveBeenCalledWith( + "agent-release", + expect.objectContaining({ + payload: expect.objectContaining({ issueId: releaseIssueId }), + idempotencyKey: buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId: releaseIssueId, + blockerIssueIds: [reviewIssueId], + blockedTransitionAt: releaseBlockedAt, + }), + }), + ); + + mockWakeup.mockClear(); + mockFindExistingIssueBlockersResolvedWakeForReadyState.mockClear(); + mockIssueService.getById.mockResolvedValue(issueRecord({ + id: releaseIssueId, + identifier: "PAP-RELEASE", + title: "Release", + status: "blocked", + assigneeAgentId: "agent-release", + blockedTransitionAt: releaseBlockedAt, + })); + mockIssueService.update.mockResolvedValue(issueRecord({ + id: releaseIssueId, + identifier: "PAP-RELEASE", + title: "Release", + status: "done", + assigneeAgentId: "agent-release", + })); + mockIssueService.listWakeableBlockedDependents.mockResolvedValue([ + { + id: qaIssueId, + assigneeAgentId: "agent-qa", + blockerIssueIds: [releaseIssueId], + blockedTransitionAt: qaBlockedAt, + }, + ]); + + expect((await request(app).patch(`/api/issues/${releaseIssueId}`).send({ status: "done" })).status).toBe(200); + await vi.waitFor(() => { + expect(mockWakeup).toHaveBeenCalledTimes(1); + }); + expect(mockWakeup).toHaveBeenCalledWith( + "agent-qa", + expect.objectContaining({ + payload: expect.objectContaining({ + issueId: qaIssueId, + resolvedBlockerIssueId: releaseIssueId, + }), + idempotencyKey: buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId: qaIssueId, + blockerIssueIds: [releaseIssueId], + blockedTransitionAt: qaBlockedAt, + }), + }), + ); + expect(mockWakeup).not.toHaveBeenCalledWith("agent-release", expect.anything()); + }); + + it("restores a blocked-and-ready dependent under the new blocked cycle key", async () => { + const parentIssueId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const childIssueId = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + const blockedTransitionAt = new Date("2026-08-03T18:00:00.000Z"); + mockIssueService.getById.mockResolvedValue(issueRecord({ + id: parentIssueId, + identifier: "PAP-200", + title: "Blocked after completion", + status: "done", + assigneeAgentId: "agent-2", + })); + mockIssueService.update.mockResolvedValue(issueRecord({ + id: parentIssueId, + identifier: "PAP-200", + title: "Blocked after completion", + status: "blocked", + assigneeAgentId: "agent-2", + blockedTransitionAt, + })); + mockIssueService.getDependencyReadiness.mockResolvedValue({ + issueId: parentIssueId, + blockerIssueIds: [childIssueId], + unresolvedBlockerIssueIds: [], + unresolvedBlockerCount: 0, + pendingFinalizeBlockerIssueIds: [], + allBlockersDone: true, + isDependencyReady: true, + }); + + const res = await request(await createApp()) + .patch(`/api/issues/${parentIssueId}`) + .send({ + status: "blocked", + blockedByIssueIds: [childIssueId], + unblockDescriptor: { owner: "board", action: "Review the restored dependency" }, + }); + + expect(res.status).toBe(200); + await vi.waitFor(() => { + expect(mockFindExistingIssueBlockersResolvedWakeForReadyState).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + dependentIssueId: parentIssueId, + blockerIssueIds: [childIssueId], + blockedTransitionAt, + }), + ); + expect(mockWakeup).toHaveBeenCalledWith( + "agent-2", + expect.objectContaining({ + reason: "issue_blockers_resolved", + idempotencyKey: buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId: parentIssueId, + blockerIssueIds: [childIssueId], + blockedTransitionAt, + }), + payload: expect.objectContaining({ + mutation: "blocked_dependency_restored", + }), + }), + ); + }); + }); + + it("does not emit a dependency wake when an unresolved or cancelled blocker remains", async () => { + mockIssueService.getById.mockResolvedValue(issueRecord({ status: "in_progress" })); + mockIssueService.update.mockResolvedValue(issueRecord({ status: "done" })); + mockIssueService.listWakeableBlockedDependents.mockResolvedValue([]); + + const res = await request(await createApp()).patch("/api/issues/issue-1").send({ status: "done" }); + expect(res.status).toBe(200); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(mockWakeup).not.toHaveBeenCalled(); + }); }); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index ec1604b950e..b5e6a3adb28 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -10557,18 +10557,21 @@ export function issueRoutes( dependentIssueId: string; resolvedBlockerIssueId: string; blockerIssueIds: string[]; + blockedTransitionAt?: Date | string | null; source: string; mutation: string; }) => { const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({ dependentIssueId: input.dependentIssueId, blockerIssueIds: input.blockerIssueIds, + blockedTransitionAt: input.blockedTransitionAt, }); try { const existingWake = await findExistingIssueBlockersResolvedWakeForReadyState(db, { companyId: issue.companyId, dependentIssueId: input.dependentIssueId, blockerIssueIds: input.blockerIssueIds, + blockedTransitionAt: input.blockedTransitionAt, }); if (existingWake) return; } catch (err) { @@ -10753,6 +10756,7 @@ export function issueRoutes( dependentIssueId: dependent.id, resolvedBlockerIssueId: issue.id, blockerIssueIds: dependent.blockerIssueIds, + blockedTransitionAt: dependent.blockedTransitionAt, source: "issue.blockers_resolved", mutation: "blocker_done", }); @@ -10780,6 +10784,7 @@ export function issueRoutes( dependentIssueId: issue.id, resolvedBlockerIssueId, blockerIssueIds: readiness.blockerIssueIds, + blockedTransitionAt: issue.blockedTransitionAt, source: "issue.blockers_restored", mutation: "blocked_dependency_restored", }); @@ -12612,16 +12617,19 @@ export function issueRoutes( dependentIssueId: string; resolvedBlockerIssueId: string; blockerIssueIds: string[]; + blockedTransitionAt?: Date | string | null; }) => { const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({ dependentIssueId: input.dependentIssueId, blockerIssueIds: input.blockerIssueIds, + blockedTransitionAt: input.blockedTransitionAt, }); try { const existingWake = await findExistingIssueBlockersResolvedWakeForReadyState(db, { companyId: currentIssue.companyId, dependentIssueId: input.dependentIssueId, blockerIssueIds: input.blockerIssueIds, + blockedTransitionAt: input.blockedTransitionAt, }); if (existingWake) return; } catch (err) { @@ -12785,6 +12793,7 @@ export function issueRoutes( dependentIssueId: dependent.id, resolvedBlockerIssueId: currentIssue.id, blockerIssueIds: dependent.blockerIssueIds, + blockedTransitionAt: dependent.blockedTransitionAt, }); } } diff --git a/server/src/services/issue-dependency-wakeups.test.ts b/server/src/services/issue-dependency-wakeups.test.ts new file mode 100644 index 00000000000..3ed05554178 --- /dev/null +++ b/server/src/services/issue-dependency-wakeups.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { + buildIssueBlockersResolvedWakeIdempotencyKey, + buildIssueBlockersResolvedWakeStateKey, + buildIssueBlockersResolvedWakeStateKeyWithoutCycle, + findExistingIssueBlockersResolvedWakeForReadyState, +} from "./issue-dependency-wakeups.js"; + +const dependentIssueId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const blockerIssueId = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const companyId = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; +const firstCycle = new Date("2026-04-01T12:00:00.000Z"); +const secondCycle = new Date("2026-08-01T09:30:00.000Z"); + +type WakeRow = { + id: string; + status: string; + idempotencyKey: string | null; + requestedAt: Date; +}; + +function dbWithWakes(rows: WakeRow[]): Db { + return { + select() { + return { + from() { + return { + where() { + return Promise.resolve(rows); + }, + }; + }, + }; + }, + } as unknown as Db; +} + +describe("buildIssueBlockersResolvedWakeStateKey", () => { + it("is identical for the same dependent, blockers, and blockedTransitionAt", () => { + const first = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: firstCycle, + }); + const second = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: firstCycle.toISOString(), + }); + expect(first).toBe(second); + expect(first).toContain(dependentIssueId); + }); + + it("changes when blockedTransitionAt changes", () => { + const first = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: firstCycle, + }); + const second = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: secondCycle, + }); + expect(first).not.toBe(second); + }); + + it("hashes a null cycle as none and differs from any timestamp", () => { + const noneKey = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: null, + }); + const omittedKey = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + }); + const datedKey = buildIssueBlockersResolvedWakeStateKey({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: firstCycle, + }); + expect(noneKey).toBe(omittedKey); + expect(noneKey).not.toBe(datedKey); + expect(noneKey).not.toBe( + buildIssueBlockersResolvedWakeStateKeyWithoutCycle({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + }), + ); + }); +}); + +describe("findExistingIssueBlockersResolvedWakeForReadyState", () => { + const readyState = { + companyId, + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: secondCycle, + }; + + it("suppresses a completed wake on the cycle-aware state key", async () => { + const cycleKey = buildIssueBlockersResolvedWakeStateKey(readyState); + const existing = await findExistingIssueBlockersResolvedWakeForReadyState( + dbWithWakes([ + { + id: "wake-cycle", + status: "completed", + idempotencyKey: cycleKey, + requestedAt: secondCycle, + }, + ]), + readyState, + ); + expect(existing?.id).toBe("wake-cycle"); + }); + + it("does not let a completed old-key wake from a previous blocked cycle suppress", async () => { + const oldKey = buildIssueBlockersResolvedWakeStateKeyWithoutCycle({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + }); + const existing = await findExistingIssueBlockersResolvedWakeForReadyState( + dbWithWakes([ + { + id: "wake-old-previous-cycle", + status: "completed", + idempotencyKey: oldKey, + requestedAt: firstCycle, + }, + ]), + readyState, + ); + expect(existing).toBeNull(); + }); + + it("suppresses a completed old-key wake requested at or after blockedTransitionAt", async () => { + const oldKey = buildIssueBlockersResolvedWakeStateKeyWithoutCycle({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + }); + const existing = await findExistingIssueBlockersResolvedWakeForReadyState( + dbWithWakes([ + { + id: "wake-old-same-cycle", + status: "completed", + idempotencyKey: oldKey, + requestedAt: secondCycle, + }, + ]), + readyState, + ); + expect(existing?.id).toBe("wake-old-same-cycle"); + }); + + it("suppresses a completed old-key wake when blockedTransitionAt is null", async () => { + const oldKey = buildIssueBlockersResolvedWakeStateKeyWithoutCycle({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + }); + const existing = await findExistingIssueBlockersResolvedWakeForReadyState( + dbWithWakes([ + { + id: "wake-old-no-cycle", + status: "completed", + idempotencyKey: oldKey, + requestedAt: firstCycle, + }, + ]), + { + companyId, + dependentIssueId, + blockerIssueIds: [blockerIssueId], + blockedTransitionAt: null, + }, + ); + expect(existing?.id).toBe("wake-old-no-cycle"); + }); + + it("suppresses an in-flight old-key wake across a later blocked cycle", async () => { + const oldKey = buildIssueBlockersResolvedWakeStateKeyWithoutCycle({ + dependentIssueId, + blockerIssueIds: [blockerIssueId], + }); + const existing = await findExistingIssueBlockersResolvedWakeForReadyState( + dbWithWakes([ + { + id: "wake-old-queued", + status: "queued", + idempotencyKey: oldKey, + requestedAt: firstCycle, + }, + ]), + readyState, + ); + expect(existing?.id).toBe("wake-old-queued"); + }); + + it("keeps legacy per-edge matching in-flight only", async () => { + const legacyKey = buildIssueBlockersResolvedWakeIdempotencyKey({ + dependentIssueId, + resolvedBlockerIssueId: blockerIssueId, + }); + const inFlight = await findExistingIssueBlockersResolvedWakeForReadyState( + dbWithWakes([ + { + id: "wake-legacy-claimed", + status: "claimed", + idempotencyKey: legacyKey, + requestedAt: firstCycle, + }, + ]), + readyState, + ); + expect(inFlight?.id).toBe("wake-legacy-claimed"); + + const completed = await findExistingIssueBlockersResolvedWakeForReadyState( + dbWithWakes([ + { + id: "wake-legacy-completed", + status: "completed", + idempotencyKey: legacyKey, + requestedAt: firstCycle, + }, + ]), + readyState, + ); + expect(completed).toBeNull(); + }); +}); diff --git a/server/src/services/issue-dependency-wakeups.ts b/server/src/services/issue-dependency-wakeups.ts index bcd565a8b4e..c40d7ff0fde 100644 --- a/server/src/services/issue-dependency-wakeups.ts +++ b/server/src/services/issue-dependency-wakeups.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, or } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { createHash } from "node:crypto"; import type { Db } from "@paperclipai/db"; import { agentWakeupRequests } from "@paperclipai/db"; @@ -19,14 +19,62 @@ const IDEMPOTENT_DEPENDENCY_WAKE_STATUSES = [ // A wake counts as "still in flight" for these statuses. The `completed` status // is not in this set on purpose. Dependency readiness is level-triggered, so a // historical completed per-edge wake must never suppress a new wake for the -// current ready state. The dedup uses this set only for the legacy per-edge key, -// to avoid a duplicate while an old-format wake is still queued or claimed. +// current ready state. The dedup uses this set only for the legacy per-edge key +// and for old no-cycle state keys that are still queued after a deploy. const IN_FLIGHT_DEPENDENCY_WAKE_STATUSES = [ "queued", "deferred_issue_execution", "claimed", ] as const; +const IDEMPOTENT_DEPENDENCY_WAKE_STATUS_SET = new Set(IDEMPOTENT_DEPENDENCY_WAKE_STATUSES); +const IN_FLIGHT_DEPENDENCY_WAKE_STATUS_SET = new Set(IN_FLIGHT_DEPENDENCY_WAKE_STATUSES); + +export type IssueBlockersResolvedWakeCycleInput = Date | string | null | undefined; + +export type IssueBlockersResolvedReadyStateInput = { + dependentIssueId: string; + blockerIssueIds: string[]; + blockedTransitionAt?: IssueBlockersResolvedWakeCycleInput; +}; + +/** + * Canonical blocked-cycle stamp for the dependency-ready state key. + * `blockedTransitionAt` is UTC ISO-8601, or `none` when the dependent has no + * recorded transition into `blocked`. + */ +export function formatIssueBlockersResolvedWakeCycle( + blockedTransitionAt: IssueBlockersResolvedWakeCycleInput, +): string { + if (blockedTransitionAt == null || blockedTransitionAt === "") return "none"; + const parsed = blockedTransitionAt instanceof Date + ? blockedTransitionAt + : new Date(blockedTransitionAt); + if (Number.isNaN(parsed.getTime())) return "none"; + return parsed.toISOString(); +} + +function uniqueSortedBlockerIssueIds(blockerIssueIds: string[]): string[] { + return [...new Set(blockerIssueIds.filter(Boolean))].sort(); +} + +function hashBlockerReadyStateDigest(sortedBlockerIssueIds: string[], cycle: string | null): string { + const payload = cycle == null + ? sortedBlockerIssueIds.join(",") + : `${sortedBlockerIssueIds.join(",")}\n${cycle}`; + return createHash("sha256").update(payload).digest("hex").slice(0, 32); +} + +function buildStateKey(dependentIssueId: string, digest: string, blockerCount: number): string { + return [ + ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, + "state", + dependentIssueId, + String(blockerCount), + digest, + ].join(":"); +} + /** * Legacy per-edge idempotency key. One key encodes a single resolved blocker * edge `issue_blockers_resolved:{dependentIssueId}:{resolvedBlockerIssueId}`. @@ -45,42 +93,96 @@ export function buildIssueBlockersResolvedWakeIdempotencyKey(input: { } /** - * Level-triggered idempotency key. One key encodes the full set of blockers that - * defines the current dependency-ready state. Two wakes for the same ready state - * share the key. A wake for an earlier partial state has a different blocker set, - * so it produces a different key and never suppresses the current wake. All three - * emit paths (route-time, finalize-time, periodic backstop) use this key so they - * share one idempotency rule. + * Pre-cycle level-triggered key. Rows written before the ready state included + * `blockedTransitionAt` hashed only the sorted blocker ids. Lookup still reads + * this format so an in-flight deploy-overlap wake can suppress a duplicate. */ -export function buildIssueBlockersResolvedWakeStateKey(input: { +export function buildIssueBlockersResolvedWakeStateKeyWithoutCycle(input: { dependentIssueId: string; blockerIssueIds: string[]; }) { - const sortedBlockerIssueIds = [...new Set(input.blockerIssueIds.filter(Boolean))].sort(); - const digest = createHash("sha256") - .update(sortedBlockerIssueIds.join(",")) - .digest("hex") - .slice(0, 32); - return [ - ISSUE_BLOCKERS_RESOLVED_WAKE_REASON, - "state", + const sortedBlockerIssueIds = uniqueSortedBlockerIssueIds(input.blockerIssueIds); + return buildStateKey( input.dependentIssueId, - String(sortedBlockerIssueIds.length), - digest, - ].join(":"); + hashBlockerReadyStateDigest(sortedBlockerIssueIds, null), + sortedBlockerIssueIds.length, + ); +} + +/** + * Level-triggered idempotency key. One key encodes the full set of blockers that + * defines the current dependency-ready state plus the dependent's current + * blocked cycle (`blockedTransitionAt`, or `none`). Two wakes for the same ready + * state share the key. A wake from an earlier blocked cycle has a different + * cycle stamp, so it produces a different key and never suppresses the current + * wake. All three emit paths (route-time, finalize-time, periodic backstop) use + * this key so they share one idempotency rule. + */ +export function buildIssueBlockersResolvedWakeStateKey(input: IssueBlockersResolvedReadyStateInput) { + const sortedBlockerIssueIds = uniqueSortedBlockerIssueIds(input.blockerIssueIds); + const cycle = formatIssueBlockersResolvedWakeCycle(input.blockedTransitionAt); + return buildStateKey( + input.dependentIssueId, + hashBlockerReadyStateDigest(sortedBlockerIssueIds, cycle), + sortedBlockerIssueIds.length, + ); +} + +function parseWakeCycleDate(blockedTransitionAt: IssueBlockersResolvedWakeCycleInput): Date | null { + if (blockedTransitionAt == null || blockedTransitionAt === "") return null; + const parsed = blockedTransitionAt instanceof Date + ? blockedTransitionAt + : new Date(blockedTransitionAt); + if (Number.isNaN(parsed.getTime())) return null; + return parsed; +} + +function wakeCoversIssueBlockersResolvedReadyState( + wake: { + status: string; + idempotencyKey: string | null; + requestedAt: Date; + }, + keys: { + cycleKey: string; + oldStateKey: string; + legacyKeys: Set; + blockedTransitionAt: Date | null; + }, +): boolean { + const idempotencyKey = wake.idempotencyKey; + if (!idempotencyKey) return false; + + if (idempotencyKey === keys.cycleKey) { + return IDEMPOTENT_DEPENDENCY_WAKE_STATUS_SET.has(wake.status); + } + + if (idempotencyKey === keys.oldStateKey) { + if (IN_FLIGHT_DEPENDENCY_WAKE_STATUS_SET.has(wake.status)) return true; + if (wake.status !== "completed") return false; + if (!keys.blockedTransitionAt) return true; + return wake.requestedAt.getTime() >= keys.blockedTransitionAt.getTime(); + } + + if (keys.legacyKeys.has(idempotencyKey)) { + return IN_FLIGHT_DEPENDENCY_WAKE_STATUS_SET.has(wake.status); + } + + return false; } /** * Find a wake that already covers the current dependency-ready state of the - * dependent issue. The check is level-triggered: + * dependent issue. The check is level-triggered and cycle-aware: * - * - The state key matches a wake in any idempotent status (including - * `completed`). This suppresses a duplicate wake for the SAME ready state and - * bounds reconciliation. - * - Each legacy per-edge key matches only a wake that is still in flight - * (`queued`, `deferred_issue_execution`, `claimed`). This prevents a duplicate - * wake while an old-format wake is still pending after a deploy, but it never - * lets a historical completed per-edge wake strand the issue. + * - The cycle-aware state key matches a wake in any idempotent status + * (including `completed`). This suppresses a duplicate for the SAME ready + * state, including the current blocked cycle. + * - The old no-cycle state key matches in-flight statuses (deploy overlap), + * or a `completed` wake whose `requestedAt` is at or after the current + * `blockedTransitionAt` (same cycle). A completed old-key wake from a + * previous cycle does not suppress. + * - Each legacy per-edge key matches only a wake that is still in flight. * * Returns the first matching wake or `null`. */ @@ -90,13 +192,12 @@ export async function findExistingIssueBlockersResolvedWakeForReadyState( companyId: string; dependentIssueId: string; blockerIssueIds: string[]; + blockedTransitionAt?: IssueBlockersResolvedWakeCycleInput; }, ) { - const stateKey = buildIssueBlockersResolvedWakeStateKey({ - dependentIssueId: input.dependentIssueId, - blockerIssueIds: input.blockerIssueIds, - }); - const legacyKeys = [ + const cycleKey = buildIssueBlockersResolvedWakeStateKey(input); + const oldStateKey = buildIssueBlockersResolvedWakeStateKeyWithoutCycle(input); + const legacyKeyList = [ ...new Set( input.blockerIssueIds .filter(Boolean) @@ -108,32 +209,31 @@ export async function findExistingIssueBlockersResolvedWakeForReadyState( ), ), ]; + const lookupKeys = [...new Set([cycleKey, oldStateKey, ...legacyKeyList])]; + const blockedTransitionAt = parseWakeCycleDate(input.blockedTransitionAt); - const stateMatch = and( - eq(agentWakeupRequests.idempotencyKey, stateKey), - inArray(agentWakeupRequests.status, [...IDEMPOTENT_DEPENDENCY_WAKE_STATUSES]), - ); - const legacyMatch = - legacyKeys.length > 0 - ? and( - inArray(agentWakeupRequests.idempotencyKey, legacyKeys), - inArray(agentWakeupRequests.status, [...IN_FLIGHT_DEPENDENCY_WAKE_STATUSES]), - ) - : null; - - return db + const rows = await db .select({ id: agentWakeupRequests.id, status: agentWakeupRequests.status, idempotencyKey: agentWakeupRequests.idempotencyKey, + requestedAt: agentWakeupRequests.requestedAt, }) .from(agentWakeupRequests) .where( and( eq(agentWakeupRequests.companyId, input.companyId), - legacyMatch ? or(stateMatch, legacyMatch) : stateMatch, + inArray(agentWakeupRequests.idempotencyKey, lookupKeys), ), - ) - .limit(1) - .then((rows) => rows[0] ?? null); + ); + + const covering = rows.find((row) => + wakeCoversIssueBlockersResolvedReadyState(row, { + cycleKey, + oldStateKey, + legacyKeys: new Set(legacyKeyList), + blockedTransitionAt, + }), + ); + return covering ?? null; } diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 39d2f2545b7..655f6c5af49 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -6571,6 +6571,7 @@ export function issueService(db: Db) { id: issues.id, assigneeAgentId: issues.assigneeAgentId, status: issues.status, + blockedTransitionAt: issues.blockedTransitionAt, }) .from(issueRelations) .innerJoin(issues, eq(issueRelations.relatedIssueId, issues.id)) @@ -6610,6 +6611,7 @@ export function issueService(db: Db) { id: candidate.id, assigneeAgentId: candidate.assigneeAgentId!, blockerIssueIds: readiness.blockerIssueIds, + blockedTransitionAt: candidate.blockedTransitionAt, })); }, diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index f715479178a..4d3f08d9aba 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -5712,6 +5712,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) companyId: issues.companyId, identifier: issues.identifier, assigneeAgentId: issues.assigneeAgentId, + blockedTransitionAt: issues.blockedTransitionAt, totalCount: sql`count(*) over()::int`, }) .from(issueRelations) @@ -5727,6 +5728,7 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) companyId: issues.companyId, identifier: issues.identifier, assigneeAgentId: issues.assigneeAgentId, + blockedTransitionAt: issues.blockedTransitionAt, totalCount: sql`count(*) over()::int`, }) .from(issues) @@ -5800,11 +5802,13 @@ export function recoveryService(db: Db, deps: { enqueueWakeup: RecoveryWakeup }) const idempotencyKey = buildIssueBlockersResolvedWakeStateKey({ dependentIssueId: candidate.id, blockerIssueIds: readiness.blockerIssueIds, + blockedTransitionAt: candidate.blockedTransitionAt, }); const existingWake = await findExistingIssueBlockersResolvedWakeForReadyState(db, { companyId, dependentIssueId: candidate.id, blockerIssueIds: readiness.blockerIssueIds, + blockedTransitionAt: candidate.blockedTransitionAt, }); if (existingWake) { result.existingWakeSkipped += 1; From 16b59c9315eb2fa2d61facc63ad3d678f32cea1e Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Sun, 23 Aug 2026 08:52:31 -0700 Subject: [PATCH 5/6] feat(adapter-utils): stream duplex bridge bodies as sequenced chunks with receive-side spill (#12006) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Agent adapters use a duplex bridge to send requests and responses across an isolated boundary > - The bridge held each request body and response body in memory on both ends > - Large bodies can exhaust memory and reduce the safe size of adapter traffic > - This pull request sends receive-side bodies as sequenced chunks and spills large bodies to disk > - The benefit is bounded memory use with strict size, order, and cleanup checks ## Linked Issues or Issue Description **What existing behavior does this improve?** The adapter-utils duplex bridge transports request and response bodies across the sandbox boundary. **Current behavior** The bridge stores each complete body in memory on both ends of the duplex channel. **Proposed behavior** The bridge sends body chunks with sequence checks. The receive side keeps bodies up to 1 MiB in memory and spills larger bodies to a temporary file. **Reason and benefit** This change reduces memory pressure and keeps malformed or oversized input on a terminal error path. **Breaking changes** The duplex frame version changes to version 2. The request and response envelopes now carry bodyByteCount, and body_chunk frames carry the body data. ## What Changed - Add version 2 body_chunk frames with 256 KiB raw slices encoded as canonical base64 text. - Add receive-side memory and spill reassembly with per-channel disk and file limits. - Reject malformed, reordered, oversized, truncated, and non-canonical body chunks. - Stream reassembled request bodies to the host forward handler with a web stream and half-duplex request. - Remove spill files on success, failure, channel death, and startup cleanup. ## Verification - Run the adapter-utils type-check. - Run the adapter-utils duplex test suite. - Run all pull request checks. - Run the Greptile review and confirm a 5/5 score with no open findings. ## Risks The wire format changes from version 1 to version 2. Older bridge peers cannot use this protocol. The receive path adds temporary file operations and cleanup paths. The implementation fails closed when a body violates size or sequence rules. ## Model Used OpenAI GPT-5 Codex, tool-enabled coding agent. The exact context-window size and reasoning mode are not exposed by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with Fixes: # / Closes # / Refs # OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub #NNN / github.com/paperclipai/paperclip URLs) - [x] My branch name describes the change (e.g. docs/..., fix/...) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../src/acpx-engine/execute.test.ts | 12 +- .../src/duplex-body-spool.test.ts | 306 +++++++++ .../adapter-utils/src/duplex-body-spool.ts | 596 ++++++++++++++++ .../duplex-bridge-broker-byte-ledger.test.ts | 121 +++- .../src/duplex-bridge-broker.test.ts | 241 +++++-- .../adapter-utils/src/duplex-bridge-broker.ts | 643 ++++++++++++------ .../src/duplex-bridge-local-e2e.test.ts | 113 ++- .../src/duplex-frame-codec.test.ts | 31 +- .../adapter-utils/src/duplex-frame-codec.ts | 82 ++- .../src/duplex-frame-vectors.json | 359 +++++++--- .../src/execution-target-sandbox.test.ts | 388 +++++++---- .../adapter-utils/src/execution-target.ts | 49 +- .../src/sandbox-callback-bridge.ts | 172 ++++- 13 files changed, 2493 insertions(+), 620 deletions(-) create mode 100644 packages/adapter-utils/src/duplex-body-spool.test.ts create mode 100644 packages/adapter-utils/src/duplex-body-spool.ts diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 6f5ae7dff6d..217e4f582d1 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -5763,8 +5763,8 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => // Wrap a started real broker in a paperclip bridge handle. The handle exposes // the same run-disposition surface the sandbox bridge exposes, so the seam runs // against the real latch and the real orderly-completion mark. - function bridgeOverBroker(fake: ReturnType) { - const broker = createDuplexBridgeBroker({ + async function bridgeOverBroker(fake: ReturnType) { + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -5891,7 +5891,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("fails a completed run when the duplex channel was lost before the completion", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake); + const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); // Latch the loss before the ACP terminal resolves. const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); @@ -5912,7 +5912,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("keeps a completed run a success when the channel stays live, and a later teardown loss is benign", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake); + const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); // No loss before the completion. const runtime = runtimeWithControlledResult(); @@ -5932,7 +5932,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("does not let a later completion or activity clear the loss latch", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle } = bridgeOverBroker(fake); + const { broker, handle } = await bridgeOverBroker(fake); // Latch the loss before the ACP terminal resolves. const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); @@ -5951,7 +5951,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("marks an orderly completion on a failed terminal so the teardown loss emits no false loss", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, markOrderlyCompletion } = bridgeOverBroker(fake); + const { broker, handle, markOrderlyCompletion } = await bridgeOverBroker(fake); // The turn fails, and no channel loss ordered before the finalization. const runtime = runtimeWithFailedResult(); diff --git a/packages/adapter-utils/src/duplex-body-spool.test.ts b/packages/adapter-utils/src/duplex-body-spool.test.ts new file mode 100644 index 00000000000..37307f2e67e --- /dev/null +++ b/packages/adapter-utils/src/duplex-body-spool.test.ts @@ -0,0 +1,306 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { DUPLEX_FRAME_VERSION, type DuplexBodyChunkFrame } from "./duplex-frame-codec.js"; +import { + DuplexBodyReceiver, + type DuplexBodyReceiverConfig, + splitBodyIntoChunkFrames, + sweepStaleSpillDirectories, +} from "./duplex-body-spool.js"; + +// Read the whole reassembled body back, whatever path it took. A memory body +// returns its buffer; a spilled body streams from disk. The test uses this only +// to assert the round-trip content, never on a hot path. +async function readBodyBytes(body: { + spilled: boolean; + createReadStream: () => NodeJS.ReadableStream; +}): Promise { + const chunks: Buffer[] = []; + for await (const chunk of body.createReadStream()) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +let spillRoot: string; + +beforeEach(async () => { + spillRoot = await fs.mkdtemp(path.join(os.tmpdir(), "duplex-spool-test-")); +}); + +afterEach(async () => { + await fs.rm(spillRoot, { recursive: true, force: true }).catch(() => undefined); +}); + +function makeReceiver(overrides: DuplexBodyReceiverConfig = {}): Promise { + return DuplexBodyReceiver.create({ spillRootDir: spillRoot, ...overrides }); +} + +const RAW = 8; + +// Feed a whole body through a receiver as an envelope plus chunk frames. Return +// the settled body handle. The caller pushes the chunks the split produces. +async function transfer( + receiver: DuplexBodyReceiver, + id: string, + body: Buffer, +): ReturnType { + const promise = receiver.begin(id, body.length); + const frames = splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION, RAW); + for (const frame of frames) { + const result = receiver.pushChunk(frame); + expect(result.ok).toBe(true); + } + return promise; +} + +describe("duplex body reassembler — round trip", () => { + it("reassembles a small body on the memory path", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const source = Buffer.from("hello duplex body"); + const body = await transfer(receiver, "req-mem", source); + expect(body.spilled).toBe(false); + expect(body.byteCount).toBe(source.length); + expect(body.readMemory().equals(source)).toBe(true); + expect((await readBodyBytes(body)).equals(source)).toBe(true); + await body.dispose(); + }); + + it("reassembles a large body on the spill path without holding it in memory", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW }); + const source = Buffer.from("abcdefghijklmnopqrstuvwxyz0123456789ABCD"); // 40 bytes + const body = await transfer(receiver, "req-spill", source); + expect(body.spilled).toBe(true); + expect(body.byteCount).toBe(source.length); + // A spilled body never loads the whole body into memory on the receive side. + expect(() => body.readMemory()).toThrow(); + expect(receiver.spillUsedBytes).toBe(source.length); + expect(receiver.spillOpenFiles).toBe(1); + expect((await readBodyBytes(body)).equals(source)).toBe(true); + await body.dispose(); + // The dispose path unlinks the file and releases the reservation. + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); + + it("completes a zero-length body at once and accepts no chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW }); + const body = await receiver.begin("req-zero", 0); + expect(body.byteCount).toBe(0); + expect(body.readMemory().length).toBe(0); + const stray = receiver.pushChunk({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id: "req-zero", + seq: 0, + data: Buffer.from("x").toString("base64"), + }); + expect(stray.ok).toBe(false); + if (!stray.ok) expect(stray.error.code).toBe("zero_body_has_chunk"); + await body.dispose(); + }); +}); + +describe("duplex body reassembler — rejection matrix", () => { + function chunk(id: string, seq: number, data: string): DuplexBodyChunkFrame { + return { version: DUPLEX_FRAME_VERSION, type: "body_chunk", id, seq, data }; + } + + it("rejects a duplicate seq", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("d", 16); + expect(receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const dup = receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 2).toString("base64"))); + expect(dup.ok).toBe(false); + if (!dup.ok) expect(dup.error.code).toBe("seq_out_of_order"); + }); + + it("rejects a gap in the seq order", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("g", 24); + expect(receiver.pushChunk(chunk("g", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const gap = receiver.pushChunk(chunk("g", 2, Buffer.alloc(RAW, 2).toString("base64"))); + expect(gap.ok).toBe(false); + if (!gap.ok) expect(gap.error.code).toBe("seq_out_of_order"); + }); + + it("rejects a reordered seq", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("r", 24); + const reorder = receiver.pushChunk(chunk("r", 1, Buffer.alloc(RAW, 1).toString("base64"))); + expect(reorder.ok).toBe(false); + if (!reorder.ok) expect(reorder.error.code).toBe("seq_out_of_order"); + }); + + it("rejects non-canonical base64", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const begun = receiver.begin("b", 8); + begun.catch(() => undefined); + // "AB==" carries non-zero bits in the final sextet, so it decodes to one + // byte and re-encodes to "AA==". The round-trip mismatch marks it not + // canonical. + const bad = receiver.pushChunk(chunk("b", 0, "AB==")); + expect(bad.ok).toBe(false); + if (!bad.ok) expect(bad.error.code).toBe("chunk_not_canonical_base64"); + }); + + it("rejects a short non-final chunk (truncation)", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("t", 24); + // A non-final chunk must be exactly RAW bytes. A 4-byte chunk that does not + // complete the 24-byte body is a truncation. + const truncation = receiver.pushChunk(chunk("t", 0, Buffer.alloc(4, 1).toString("base64"))); + expect(truncation.ok).toBe(false); + if (!truncation.ok) expect(truncation.error.code).toBe("chunk_wrong_size"); + }); + + it("rejects an empty non-final chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("e", 16); + const empty = receiver.pushChunk(chunk("e", 0, "")); + expect(empty.ok).toBe(false); + if (!empty.ok) expect(empty.error.code).toBe("empty_chunk"); + }); + + it("rejects an overrun past the declared body size", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + // A 12-byte body takes one full RAW chunk, then a final chunk of 4 bytes. A + // second full RAW chunk overruns the declared size. + void receiver.begin("o", 12); + expect(receiver.pushChunk(chunk("o", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const overrun = receiver.pushChunk(chunk("o", 1, Buffer.alloc(RAW, 2).toString("base64"))); + expect(overrun.ok).toBe(false); + if (!overrun.ok) expect(overrun.error.code).toBe("body_overrun"); + }); + + it("rejects an over-size final chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("os", 40); + const over = receiver.pushChunk(chunk("os", 0, Buffer.alloc(RAW + 4, 1).toString("base64"))); + expect(over.ok).toBe(false); + if (!over.ok) expect(over.error.code).toBe("chunk_wrong_size"); + }); + + it("rejects a chunk that arrives with no matching envelope", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const orphan = receiver.pushChunk(chunk("missing", 0, Buffer.alloc(RAW, 1).toString("base64"))); + expect(orphan.ok).toBe(false); + if (!orphan.ok) expect(orphan.error.code).toBe("chunk_without_envelope"); + }); + + it("rejects a chunk that arrives after completion", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const promise = receiver.begin("c", RAW); + expect(receiver.pushChunk(chunk("c", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + await promise; + const late = receiver.pushChunk(chunk("c", 1, Buffer.alloc(RAW, 2).toString("base64"))); + expect(late.ok).toBe(false); + if (!late.ok) expect(late.error.code).toBe("chunk_after_completion"); + }); +}); + +describe("duplex body reassembler — spill caps fail closed", () => { + it("fails closed when a write exceeds the per-channel byte cap and unlinks the partial file", async () => { + // A 5-byte cap admits the first RAW=8 chunk write? No: reserve 8 > cap 5, so + // the first write fails closed. Use a cap that admits one chunk but not two. + const receiver = await makeReceiver({ + spillThresholdBytes: 4, + rawChunkBytes: RAW, + channelSpillByteCap: RAW + 2, + }); + const promise = receiver.begin("cap", 24); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // The second chunk write reserves past the byte cap. + receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 1, data: Buffer.alloc(RAW, 2).toString("base64") }); + await expect(promise).rejects.toMatchObject({ code: "spill_cap_exceeded" }); + // The partial file is gone and the reservation released. + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); + + it("fails closed when the open-file cap is reached", async () => { + const receiver = await makeReceiver({ + spillThresholdBytes: 4, + rawChunkBytes: RAW, + channelSpillFileCap: 1, + }); + // The first spill body claims the one file slot and stays open. The channel + // dispose rejects this in-flight body, so attach the catch at once. + const first = receiver.begin("f1", 24); + first.catch(() => undefined); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "f1", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // The second spill body cannot claim a file slot. + await expect(receiver.begin("f2", 24)).rejects.toMatchObject({ code: "spill_cap_exceeded" }); + expect(receiver.spillOpenFiles).toBe(1); + // The dispose of the in-flight first body rejects its begin promise with a + // terminal channel-closed error and frees the file slot. + await receiver.disposeBody("f1"); + await expect(first).rejects.toMatchObject({ code: "channel_closed" }); + expect(receiver.spillOpenFiles).toBe(0); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — lifecycle cleanup", () => { + it("releases the reservation and unlinks the file on channel death", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW }); + // The channel death rejects this in-flight body, so attach the catch at once. + const dying = receiver.begin("death", 24); + dying.catch(() => undefined); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "death", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // Wait for the write to land, so the file exists before the channel dies. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(receiver.spillOpenFiles).toBe(1); + await receiver.disposeAll(); + await expect(dying).rejects.toMatchObject({ code: "channel_closed" }); + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — spill file safety", () => { + it("creates the spill directory with mode 0700 and each spill file with mode 0600", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW }); + const dirStat = await fs.stat(receiver.spillDir); + expect(dirStat.mode & 0o777).toBe(0o700); + // The final destroy rejects this in-flight body, so attach the catch at once. + const pending = receiver.begin("mode", 8); + pending.catch(() => undefined); + receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "mode", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }); + await new Promise((resolve) => setTimeout(resolve, 20)); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(1); + // The file name is random, so no frame id rides the path. + expect(entries[0]).not.toContain("mode"); + const fileStat = await fs.stat(path.join(receiver.spillDir, entries[0])); + expect(fileStat.mode & 0o777).toBe(0o600); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — startup sweep", () => { + it("removes a stale spill directory from a dead process but keeps a live one", async () => { + const deadPid = 999999; // A pid that is almost never alive. + const staleDir = path.join(spillRoot, `paperclip-duplex-spill-${deadPid}-stale`); + await fs.mkdir(staleDir); + await fs.writeFile(path.join(staleDir, "leftover.body"), "x"); + const liveDir = path.join(spillRoot, `paperclip-duplex-spill-${process.pid}-live`); + await fs.mkdir(liveDir); + await sweepStaleSpillDirectories(spillRoot); + await expect(fs.stat(staleDir)).rejects.toThrow(); + // The sweep keeps a directory that belongs to a live process. + await expect(fs.stat(liveDir)).resolves.toBeDefined(); + }); +}); diff --git a/packages/adapter-utils/src/duplex-body-spool.ts b/packages/adapter-utils/src/duplex-body-spool.ts new file mode 100644 index 00000000000..576caca5afe --- /dev/null +++ b/packages/adapter-utils/src/duplex-body-spool.ts @@ -0,0 +1,596 @@ +/** + * Receive-side body reassembly and spill for the sandbox duplex channel. + * + * The duplex codec splits one request body or one response body into an envelope + * frame plus a sequence of `body_chunk` frames. This module reassembles those + * chunks back into one body. The receiver reads `bodyByteCount` from the envelope + * first, then chooses one of two paths before it reads the first chunk: + * - memory path: a body at or below the spill threshold stays in a memory + * buffer. + * - spill path: a body above the spill threshold streams to a temporary file, + * so the receiver never holds a whole large body in memory. + * + * The spill path bounds the disk a single channel can consume. A per-channel + * arena caps the bytes on disk and the number of open spill files. The receiver + * reserves a file slot and reserves bytes before it opens a file and before each + * write. A reserve that exceeds a cap fails closed: it rejects the body as a + * resource error and unlinks any partial file. + * + * The reassembler treats every frame as untrusted. It rejects a bad chunk as a + * loud, terminal protocol error: a non-canonical base64 payload, a wrong chunk + * size, an out-of-order `seq`, a total that does not equal `bodyByteCount`, or a + * chunk that arrives for a completed or unknown body. + */ + +import { randomBytes } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { promises as fs } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { Readable } from "node:stream"; + +import { + DUPLEX_BODY_CHUNK_RAW_BYTES, + type DuplexBodyChunkFrame, +} from "./duplex-frame-codec.js"; + +/** + * The default spill threshold, in bytes. A body at or below this size stays in + * memory. A body above it spills to disk. The value is 1 MiB. It stays + * injectable so a test can lower it and exercise the spill path. + */ +export const DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES = 1024 * 1024; + +/** + * The default per-channel spill byte cap. The receiver keeps at most this many + * bytes on disk at one time for one channel. The value is 64 MiB. It is about + * 0.64% of the default 10 GiB sandbox disk and stays far below the security + * ceiling of 25% of the disk quota. It stays injectable so a test can lower it. + */ +export const DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP = 64 * 1024 * 1024; + +/** + * The default per-channel open-file cap. The receiver keeps at most this many + * spill files open at one time for one channel. The value derives from the byte + * cap and the spill threshold: 64 MiB divided by the 1 MiB spill floor gives 64. + * It stays injectable so a test can lower it. + */ +export const DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP = 64; + +/** The prefix of every spill directory name. The startup sweep matches it. */ +const SPILL_DIR_PREFIX = "paperclip-duplex-spill-"; + +/** The config a caller passes to build a {@link DuplexBodyReceiver}. */ +export interface DuplexBodyReceiverConfig { + /** The spill threshold in bytes. Defaults to {@link DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES}. */ + spillThresholdBytes?: number; + /** The fixed raw chunk size in bytes. Defaults to {@link DUPLEX_BODY_CHUNK_RAW_BYTES}. */ + rawChunkBytes?: number; + /** The per-channel spill byte cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP}. */ + channelSpillByteCap?: number; + /** The per-channel open-file cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP}. */ + channelSpillFileCap?: number; + /** The root directory that holds the spill directory. Defaults to the OS temp dir. */ + spillRootDir?: string; + /** A diagnostic sink for a non-fatal cleanup note. The receiver names no body id here. */ + logger?: (message: string) => void; +} + +/** The reason the reassembler rejected a body. Each reason is a terminal protocol error. */ +export type DuplexBodyErrorCode = + | "chunk_without_envelope" + | "chunk_after_completion" + | "seq_out_of_order" + | "chunk_not_canonical_base64" + | "chunk_wrong_size" + | "empty_chunk" + | "body_overrun" + | "zero_body_has_chunk" + | "spill_cap_exceeded" + | "spill_write_failed" + | "path_escape" + | "channel_closed"; + +/** A reassembly error. The receiver returns it; it never leaks a raw path or a raw provider string. */ +export class DuplexBodyError extends Error { + readonly code: DuplexBodyErrorCode; + constructor(code: DuplexBodyErrorCode, message: string) { + super(message); + this.name = "DuplexBodyError"; + this.code = code; + } +} + +/** + * One reassembled body. The memory path holds the whole body in a buffer. The + * spill path holds a temporary file. A caller reads the memory buffer, or opens + * a fresh read stream, then disposes the body once. + */ +export interface ReassembledBody { + /** The raw byte length of the body. */ + readonly byteCount: number; + /** True when the body spilled to disk. */ + readonly spilled: boolean; + /** + * The whole body as a buffer. The memory path returns it. The spill path + * throws, because the spill path must never load a whole large body into + * memory on the receive side. A caller streams a spilled body instead. + */ + readMemory(): Buffer; + /** A fresh read stream over the body. The memory path streams one buffer; the spill path streams the file. */ + createReadStream(): Readable; + /** Release the body. The spill path unlinks the file and releases the reservation exactly once. */ + dispose(): Promise; +} + +/** + * The per-channel arena that bounds the spill disk use. It tracks the bytes on + * disk and the open files, and it owns the spill directory. It hands out + * reservations and takes them back. The reservation math is synchronous, so a + * concurrent reserve never races past a cap. + */ +class SpillArena { + usedBytes = 0; + openFiles = 0; + constructor( + readonly dir: string, + private readonly byteCap: number, + private readonly fileCap: number, + ) {} + + /** Reserve one open-file slot. Return false when the file cap is already at the limit. */ + reserveFile(): boolean { + if (this.openFiles >= this.fileCap) return false; + this.openFiles += 1; + return true; + } + + /** Release one open-file slot. */ + releaseFile(): void { + if (this.openFiles > 0) this.openFiles -= 1; + } + + /** Reserve `n` bytes of disk. Return false when the byte cap would be exceeded. */ + reserveBytes(n: number): boolean { + if (this.usedBytes + n > this.byteCap) return false; + this.usedBytes += n; + return true; + } + + /** Release `n` bytes of disk. */ + releaseBytes(n: number): void { + this.usedBytes -= n; + if (this.usedBytes < 0) this.usedBytes = 0; + } +} + +/** The internal state of one in-flight body. */ +interface ActiveBody { + bodyByteCount: number; + nextSeq: number; + receivedBytes: number; + spilled: boolean; + /** Memory-path accumulation. Empty for the spill path. */ + memoryChunks: Buffer[]; + /** The spill file path. Null for the memory path. */ + filePath: string | null; + /** The open spill file handle. Null for the memory path or before the first write. */ + handle: FileHandle | null; + /** The bytes this body reserved in the arena. The dispose path releases exactly this many. */ + reservedBytes: number; + /** True when the body holds an arena file slot that the dispose path must release. */ + holdsFileSlot: boolean; + /** True once the dispose path ran, so it runs its release once. */ + disposed: boolean; + /** Resolve the completion promise the caller awaits. */ + resolve: (body: ReassembledBody) => void; + /** Reject the completion promise on a terminal error. */ + reject: (error: DuplexBodyError) => void; + /** Serialize the spill writes, so the chunks land on disk in `seq` order. */ + writeChain: Promise; + /** True once the body settled (resolved or rejected), so a late chunk never double-settles. */ + settled: boolean; +} + +/** The synchronous result of one `pushChunk` call. */ +export type DuplexBodyPushResult = + | { ok: true } + | { ok: false; error: DuplexBodyError }; + +/** + * A per-channel receive-side body reassembler. One instance serves one channel. + * The caller feeds it one envelope per body, then the `body_chunk` frames for + * that body. The reassembler reassembles the body on the memory path or the + * spill path and settles one completion promise per body. + */ +export class DuplexBodyReceiver { + private readonly spillThresholdBytes: number; + private readonly rawChunkBytes: number; + private readonly arena: SpillArena; + private readonly logger?: (message: string) => void; + private readonly active = new Map(); + + private constructor(arena: SpillArena, config: DuplexBodyReceiverConfig) { + this.arena = arena; + this.spillThresholdBytes = + config.spillThresholdBytes ?? DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES; + this.rawChunkBytes = config.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; + this.logger = config.logger; + } + + /** + * Build a receiver. The receiver owns the spill directory. It creates the + * directory with `mkdtemp` and mode 0700, then sweeps stale spill directories + * from a prior process. The sweep runs and awaits before the receiver serves a + * body, so no caller races a half-swept directory. + */ + static async create(config: DuplexBodyReceiverConfig = {}): Promise { + const root = config.spillRootDir ?? os.tmpdir(); + await sweepStaleSpillDirectories(root, config.logger); + const dir = await fs.mkdtemp(path.join(root, `${SPILL_DIR_PREFIX}${process.pid}-`)); + // `mkdtemp` creates the directory with mode 0700 on a POSIX host by default, + // but a permissive umask can widen it. Set the mode so the owner alone reads, + // writes, and lists the spill directory. + await fs.chmod(dir, 0o700); + const arena = new SpillArena( + dir, + config.channelSpillByteCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP, + config.channelSpillFileCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP, + ); + return new DuplexBodyReceiver(arena, config); + } + + /** The spill directory this receiver owns. A test reads it to assert containment and cleanup. */ + get spillDir(): string { + return this.arena.dir; + } + + /** The bytes on disk across all in-flight bodies for this channel. A test asserts the reservation. */ + get spillUsedBytes(): number { + return this.arena.usedBytes; + } + + /** The open spill files for this channel. A test asserts the file-slot reservation. */ + get spillOpenFiles(): number { + return this.arena.openFiles; + } + + /** + * Begin one body. The caller passes the envelope `id` and `bodyByteCount`. The + * receiver chooses the memory path or the spill path from `bodyByteCount` + * before the first chunk arrives. A `bodyByteCount` of 0 completes at once with + * an empty body and accepts no chunk. The returned promise resolves with the + * reassembled body, or rejects with a {@link DuplexBodyError} on a terminal + * protocol or resource error. + */ + begin(id: string, bodyByteCount: number): Promise { + return new Promise((resolve, reject) => { + // A repeated envelope id for an in-flight body is a protocol violation. The + // caller must not begin the same id twice. Reject the new begin and leave + // the in-flight body untouched. + if (this.active.has(id)) { + reject(new DuplexBodyError("chunk_after_completion", "duplicate body envelope id")); + return; + } + const body: ActiveBody = { + bodyByteCount, + nextSeq: 0, + receivedBytes: 0, + spilled: bodyByteCount > this.spillThresholdBytes, + memoryChunks: [], + filePath: null, + handle: null, + reservedBytes: 0, + holdsFileSlot: false, + disposed: false, + resolve, + reject, + writeChain: Promise.resolve(), + settled: false, + }; + this.active.set(id, body); + if (bodyByteCount === 0) { + // The zero-length body completes at once. It accepts no chunk, so the + // receiver keeps the id in a completed state to reject a stray chunk. + this.complete(id, body); + return; + } + if (body.spilled) { + // Reserve one file slot before the receiver opens the spill file. A + // reserve past the file cap fails closed at once. + if (!this.arena.reserveFile()) { + this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill file cap reached")); + return; + } + body.holdsFileSlot = true; + // Open the spill file with a random name and mode 0600. The name never + // derives from the frame id, so no id rides a path. Verify the resolved + // path stays inside the spill directory before the receiver writes. + this.openSpillFile(id, body); + } + }); + } + + /** + * Push one `body_chunk` frame. The receiver validates the chunk against the + * in-flight body, then appends it on the memory path or the spill path. The + * return value reports a synchronous validation outcome. A spill write failure + * or a completion rejects the begin promise instead. + */ + pushChunk(frame: DuplexBodyChunkFrame): DuplexBodyPushResult { + const body = this.active.get(frame.id); + if (!body) { + return fail("chunk_without_envelope", "body_chunk arrived with no matching envelope"); + } + if (body.bodyByteCount === 0) { + return fail("zero_body_has_chunk", "a zero-length body accepts no body_chunk"); + } + if (body.settled) { + return fail("chunk_after_completion", "body_chunk arrived after the body completed"); + } + if (frame.seq !== body.nextSeq) { + return fail("seq_out_of_order", "body_chunk seq is a duplicate, a gap, or a reorder"); + } + // Decode the base64 payload and confirm it is canonical. A non-canonical + // payload re-encodes to a different string, so the round-trip catches it. + const decoded = Buffer.from(frame.data, "base64"); + if (decoded.toString("base64") !== frame.data) { + return fail("chunk_not_canonical_base64", "body_chunk data is not canonical base64"); + } + if (decoded.length === 0) { + return fail("empty_chunk", "body_chunk is empty"); + } + const nextReceived = body.receivedBytes + decoded.length; + if (nextReceived > body.bodyByteCount) { + return fail("body_overrun", "body_chunk overruns the declared body size"); + } + const isFinal = nextReceived === body.bodyByteCount; + if (!isFinal && decoded.length !== this.rawChunkBytes) { + // A non-final chunk must be exactly the fixed raw slice size. A short + // non-final chunk is a truncation; an over-size chunk is a framing fault. + return fail("chunk_wrong_size", "a non-final body_chunk is not the fixed slice size"); + } + if (decoded.length > this.rawChunkBytes) { + // The final chunk must also stay within the fixed raw slice size. + return fail("chunk_wrong_size", "a body_chunk exceeds the fixed slice size"); + } + body.nextSeq += 1; + body.receivedBytes = nextReceived; + if (body.spilled) { + this.appendSpill(frame.id, body, decoded, isFinal); + } else { + body.memoryChunks.push(decoded); + if (isFinal) this.complete(frame.id, body); + } + return { ok: true }; + } + + /** + * Dispose one body by id, whatever its state. The dispose path unlinks the + * spill file and releases the arena reservation exactly once. The caller runs + * it on every settle path and on channel death, so a spill file never lingers. + */ + async disposeBody(id: string): Promise { + const body = this.active.get(id); + if (!body) return; + this.active.delete(id); + await this.release(body); + } + + /** + * Dispose every in-flight body. The caller runs it on channel death, so a + * channel that dies mid-body leaves no spill file behind. + */ + async disposeAll(): Promise { + const ids = [...this.active.keys()]; + await Promise.all(ids.map((id) => this.disposeBody(id))); + } + + /** + * Remove the spill directory this receiver owns. The caller runs it after it + * disposes every body, on channel shutdown. + */ + async destroy(): Promise { + await this.disposeAll(); + await fs.rm(this.arena.dir, { recursive: true, force: true }).catch(() => undefined); + } + + private openSpillFile(id: string, body: ActiveBody): void { + const name = `${randomBytes(24).toString("hex")}.body`; + const filePath = path.join(this.arena.dir, name); + // Path containment. The resolved file path must stay inside the spill + // directory. A generated random name never escapes, but the check is defense + // in depth against a future name scheme. + const resolvedDir = path.resolve(this.arena.dir); + const resolvedFile = path.resolve(filePath); + if (resolvedFile !== path.join(resolvedDir, name) || !resolvedFile.startsWith(resolvedDir + path.sep)) { + this.settleReject(id, body, new DuplexBodyError("path_escape", "spill path escapes the spill directory")); + return; + } + body.filePath = filePath; + // Open the file once, exclusive, mode 0600. The write chain appends each + // chunk in order to this one handle. + body.writeChain = body.writeChain.then(async () => { + body.handle = await fs.open(filePath, "wx", 0o600); + }); + body.writeChain.catch((error) => { + this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error))); + }); + } + + private appendSpill(id: string, body: ActiveBody, decoded: Buffer, isFinal: boolean): void { + // Reserve the bytes before the write. A reserve past the byte cap fails + // closed: reject the body and unlink the partial file. + if (!this.arena.reserveBytes(decoded.length)) { + this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill byte cap reached")); + return; + } + body.reservedBytes += decoded.length; + body.writeChain = body.writeChain.then(async () => { + if (body.settled) return; + if (!body.handle) throw new Error("spill file handle is not open"); + await body.handle.write(decoded); + if (isFinal) { + await body.handle.close(); + body.handle = null; + this.complete(id, body); + } + }); + body.writeChain.catch((error) => { + if (error instanceof DuplexBodyError) return; + this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error))); + }); + } + + private complete(id: string, body: ActiveBody): void { + if (body.settled) return; + body.settled = true; + const receiver = this; + const handle: ReassembledBody = { + byteCount: body.bodyByteCount, + spilled: body.spilled, + readMemory(): Buffer { + if (body.spilled) { + throw new DuplexBodyError("spill_write_failed", "a spilled body has no in-memory buffer"); + } + return Buffer.concat(body.memoryChunks); + }, + createReadStream(): Readable { + if (body.spilled && body.filePath) return createReadStream(body.filePath); + return Readable.from(Buffer.concat(body.memoryChunks)); + }, + async dispose(): Promise { + await receiver.disposeBody(id); + }, + }; + body.resolve(handle); + } + + private settleReject(id: string, body: ActiveBody, error: DuplexBodyError): void { + if (body.settled) return; + body.settled = true; + // Release the reservation and unlink the partial file before the reject, so + // a rejected body never keeps a spill file or a reservation. The reject fires + // after the release, so a caller that awaits the reject sees a clean arena. + void this.disposeBody(id).finally(() => body.reject(error)); + } + + private async release(body: ActiveBody): Promise { + if (body.disposed) return; + body.disposed = true; + // Wait for the pending writes to settle, so the unlink never races an open + // handle. Swallow a write error here; the reject path already reported it. + await body.writeChain.catch(() => undefined); + if (body.handle) { + await body.handle.close().catch(() => undefined); + body.handle = null; + } + if (body.filePath) { + await fs.rm(body.filePath, { force: true }).catch((error) => { + this.logger?.(`Duplex spill unlink failed: ${errorText(error)}`); + }); + } + if (body.reservedBytes > 0) { + this.arena.releaseBytes(body.reservedBytes); + body.reservedBytes = 0; + } + if (body.holdsFileSlot) { + this.arena.releaseFile(); + body.holdsFileSlot = false; + } + // A body that the channel disposes before it settles must not leave its + // begin promise pending. Reject it with a terminal channel-closed error, so + // a caller that awaits the body sees the channel death. A body that already + // settled keeps its outcome: `complete` resolves it, and `settleReject` sets + // `settled` before it calls dispose, so this guard never double-settles. + if (!body.settled) { + body.settled = true; + body.reject( + new DuplexBodyError("channel_closed", "the channel closed before the body completed"), + ); + } + } +} + +function fail(code: DuplexBodyErrorCode, message: string): DuplexBodyPushResult { + return { ok: false, error: new DuplexBodyError(code, message) }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Split a whole body buffer into `body_chunk` frames. The send side reads the + * source body once, then calls this to build the frames. Each frame carries one + * raw slice of `rawChunkBytes` as base64 text, except the final frame, which + * carries the remaining bytes. A zero-length body yields no frame. + */ +export function splitBodyIntoChunkFrames( + id: string, + body: Buffer, + version: number, + rawChunkBytes: number = DUPLEX_BODY_CHUNK_RAW_BYTES, +): DuplexBodyChunkFrame[] { + const frames: DuplexBodyChunkFrame[] = []; + let seq = 0; + for (let offset = 0; offset < body.length; offset += rawChunkBytes) { + const slice = body.subarray(offset, offset + rawChunkBytes); + frames.push({ + version, + type: "body_chunk", + id, + seq, + data: slice.toString("base64"), + }); + seq += 1; + } + return frames; +} + +/** + * Sweep stale spill directories from a prior process. The receiver runs it at + * startup, before it serves a body. It removes a spill directory whose embedded + * pid is not this process and is not a live process, so a crash that left a + * directory behind does not leak disk. It never removes a directory of a live + * process, so a concurrent channel keeps its own spill directory. + */ +export async function sweepStaleSpillDirectories( + root: string, + logger?: (message: string) => void, +): Promise { + const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); + await Promise.all( + entries.map(async (entry) => { + if (!entry.isDirectory() || !entry.name.startsWith(SPILL_DIR_PREFIX)) return; + const pid = parsePidFromSpillDir(entry.name); + if (pid !== null && pid !== process.pid && isProcessAlive(pid)) return; + if (pid === process.pid) return; + await fs.rm(path.join(root, entry.name), { recursive: true, force: true }).catch((error) => { + logger?.(`Duplex stale spill sweep failed: ${errorText(error)}`); + }); + }), + ); +} + +/** Read the pid segment from a spill directory name, or null when it is absent. */ +function parsePidFromSpillDir(name: string): number | null { + const rest = name.slice(SPILL_DIR_PREFIX.length); + const dash = rest.indexOf("-"); + const pidText = dash === -1 ? rest : rest.slice(0, dash); + const pid = Number(pidText); + return Number.isInteger(pid) && pid > 0 ? pid : null; +} + +/** Return true when a process with the pid is alive. A signal 0 probes without a real signal. */ +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM means the process exists but this process cannot signal it. ESRCH + // means no such process. Treat only ESRCH as dead. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} diff --git a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts index 78780c9ff59..ff901dd1d98 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts @@ -16,6 +16,7 @@ import { type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; /** @@ -35,41 +36,83 @@ interface PendingForward { settled: boolean; } +/** + * One reassembled response the broker wrote back. The broker writes a response as + * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames + * that carry the body. The harness reassembles the body and exposes it as `body`. + */ +type ReassembledResponse = DuplexResponseFrame & { body: string }; + +/** One request the harness feeds: the envelope frame plus its raw body text. */ +interface RequestInput { + frame: DuplexRequestFrame; + bodyText: string; +} + /** The in-memory channel plus the levers a test uses to drive the broker. */ interface FakeChannelHarness { channel: CommandManagedDuplexChannel; - feed: (frame: DuplexRequestFrame) => void; + feed: (input: RequestInput) => void; exit: () => void; - responses: DuplexResponseFrame[]; + responses: ReassembledResponse[]; forwards: PendingForward[]; resolveForward: (id: string, body?: string) => void; } -/** Build one valid request frame. */ -function requestFrame(id: string, method = "POST"): DuplexRequestFrame { +/** Build one valid request: the envelope carries `bodyByteCount`, the body rides body_chunk frames. */ +function requestFrame(id: string, method = "POST"): RequestInput { + const bodyText = JSON.stringify({ id }); return { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ id }), + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id, + method, + path: `/api/issues/${id}`, + query: "", + headers: { "content-type": "application/json" }, + bodyByteCount: Buffer.byteLength(bodyText, "utf8"), + }, + bodyText, }; } function createFakeChannelHarness(): FakeChannelHarness { - const responses: DuplexResponseFrame[] = []; + const responses: ReassembledResponse[] = []; const forwards: PendingForward[] = []; const writtenDecoder = new DuplexFrameDecoder(); + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); let dataListener: ((chunk: string) => void) | null = null; let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; const channel: CommandManagedDuplexChannel = { write: (data) => { for (const result of writtenDecoder.push(data)) { - if (result.ok && result.frame.type === "response") responses.push(result.frame); + if (!result.ok) continue; + const frame = result.frame; + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + responses.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) continue; + const decoded = Buffer.from(frame.data, "base64"); + assembly.received += decoded.length; + assembly.chunks.push(decoded); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + responses.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } + } } }, onData: (listener) => { @@ -84,9 +127,18 @@ function createFakeChannelHarness(): FakeChannelHarness { return { channel, - feed: (frame) => { + feed: ({ frame, bodyText }) => { if (!dataListener) throw new Error("The broker did not bind the data listener."); dataListener(encodeDuplexFrame(frame)); + if (bodyText.length > 0) { + for (const chunk of splitBodyIntoChunkFrames( + frame.id, + Buffer.from(bodyText, "utf8"), + DUPLEX_FRAME_VERSION, + )) { + dataListener(encodeDuplexFrame(chunk)); + } + } }, exit: () => { if (!exitListener) throw new Error("The broker did not bind the exit listener."); @@ -111,10 +163,12 @@ function controllableForward(harness: FakeChannelHarness) { }); } -/** Wait for the microtask queue to drain, so a settled promise runs its handlers. */ +/** + * Wait for the microtasks and one macrotask to settle, so the broker reassembles + * a request body, dispatches its forward, and runs each settled promise handler. + */ async function flush(): Promise { - await Promise.resolve(); - await Promise.resolve(); + await new Promise((resolve) => setTimeout(resolve, 0)); } describe("duplex bridge broker aggregate byte ledger", () => { @@ -130,7 +184,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => { const harness = createFakeChannelHarness(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -139,12 +193,15 @@ describe("duplex bridge broker aggregate byte ledger", () => { broker.start(); harness.feed(requestFrame("req-1")); - // The dispatch retains three tokens: the raw frame, the normalized payload, - // and the no-replay set entry. + // The dispatch retains three tokens at the request envelope: the raw frame, the + // normalized payload, and the no-replay set entry. The reservation is + // synchronous, so the charge holds before the body reassembles. expect(ledger.liveTokenCount).toBe(3); expect(ledger.bytesInUse).toBeGreaterThan(0); const chargedInFlight = ledger.bytesInUse; + // The broker reassembles the request body, then dispatches the forward. + await flush(); harness.resolveForward("req-1"); await flush(); @@ -164,7 +221,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => { const harness = createFakeChannelHarness(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // Squeeze the nested budgets so the response timer fires quickly while the @@ -202,7 +259,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { // can size a ceiling one byte below it and force the reservation to fail. const probeHarness = createFakeChannelHarness(); const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const measuringBroker = createDuplexBridgeBroker({ + const measuringBroker = await createDuplexBridgeBroker({ channel: probeHarness.channel, forwardRequest: controllableForward(probeHarness), duplexAggregateByteLedger: measured, @@ -213,7 +270,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { await measuringBroker.close(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -238,7 +295,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { // admitted: raise the ceiling and re-feed. const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); const roomyHarness = createFakeChannelHarness(); - const roomyBroker = createDuplexBridgeBroker({ + const roomyBroker = await createDuplexBridgeBroker({ channel: roomyHarness.channel, forwardRequest: controllableForward(roomyHarness), duplexAggregateByteLedger: roomyLedger, @@ -246,13 +303,14 @@ describe("duplex bridge broker aggregate byte ledger", () => { brokers.push(roomyBroker); roomyBroker.start(); roomyHarness.feed(requestFrame("req-1")); + await flush(); expect(roomyHarness.forwards.length).toBe(1); }); it("releases every retained token on a terminal channel loss", async () => { const harness = createFakeChannelHarness(); const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -262,8 +320,14 @@ describe("duplex bridge broker aggregate byte ledger", () => { harness.feed(requestFrame("req-1")); harness.feed(requestFrame("req-2")); + // The reservation is synchronous, so both dispatches charge three tokens each + // at the request envelope, before the bodies reassemble. expect(ledger.liveTokenCount).toBe(6); + // Let both requests reassemble and start their forwards, so the loss below + // transfers live forwards to the orphan registry. + await flush(); + // The channel exits mid-flight. The broker latches the loss, transfers the // in-flight forwards to the orphan registry, and releases the seen-id tokens. harness.exit(); @@ -294,7 +358,7 @@ describe("duplex bridge broker aggregate byte ledger", () => { }, }, }); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), duplexAggregateByteLedger: ledger, @@ -303,6 +367,9 @@ describe("duplex bridge broker aggregate byte ledger", () => { broker.start(); harness.feed(requestFrame("req-1")); + // Let the request reassemble and start its forward before the close, so the + // close orphans a live forward that settles afterward. + await flush(); await broker.close(); // The close aborted the in-flight forward and orphaned it. The forward now // settles after the close: its finally owner releases the request tokens once. diff --git a/packages/adapter-utils/src/duplex-bridge-broker.test.ts b/packages/adapter-utils/src/duplex-bridge-broker.test.ts index 6c87a1ae877..346c0cbf392 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.test.ts @@ -14,6 +14,7 @@ import { type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import { createDuplexTelemetry, DUPLEX_COUNTER_LOSS_TOTAL, @@ -42,13 +43,27 @@ interface PendingForward { settled: boolean; } +/** + * One reassembled response the broker wrote back. The broker writes a response as + * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames + * that carry the body. The harness reassembles the body and exposes it as `body`, + * so a test asserts the response the same way it did with the one-frame model. + */ +type ReassembledResponse = DuplexResponseFrame & { body: string }; + +/** One request the harness feeds: the envelope frame plus its raw body text. */ +interface RequestInput { + frame: DuplexRequestFrame; + bodyText: string; +} + /** The in-memory channel plus the levers a test uses to drive the broker. */ interface FakeChannelHarness { channel: CommandManagedDuplexChannel; - /** Push one request frame into the broker read path. */ - feed: (frame: DuplexRequestFrame) => void; - /** The response frames the broker wrote back, in order. */ - responses: DuplexResponseFrame[]; + /** Push one request (its envelope frame and its body_chunk frames) into the broker read path. */ + feed: (input: RequestInput) => void; + /** The reassembled responses the broker wrote back, in order. */ + responses: ReassembledResponse[]; /** Every forward call the broker made, in order. */ forwards: PendingForward[]; /** Resolve the newest unsettled forward for one id with a 200 result. */ @@ -57,36 +72,71 @@ interface FakeChannelHarness { resolveAll: () => void; } -/** Build one valid request frame with distinctive, secret-looking fields. */ -function requestFrame(id: string, method = "POST"): DuplexRequestFrame { +/** + * Build one valid request with distinctive, secret-looking fields. The envelope + * carries `bodyByteCount`, and the harness rides the body on `body_chunk` frames. + */ +function requestFrame( + id: string, + method = "POST", + bodyText: string = JSON.stringify({ secret: "secret-request-body" }), +): RequestInput { return { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "?secret-query=leak", - headers: { authorization: "Bearer super-secret-provider-token" }, - body: JSON.stringify({ secret: "secret-request-body" }), + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id, + method, + path: `/api/issues/${id}`, + query: "?secret-query=leak", + headers: { authorization: "Bearer super-secret-provider-token" }, + bodyByteCount: Buffer.byteLength(bodyText, "utf8"), + }, + bodyText, }; } /** * Build the in-memory channel harness. The channel keeps the broker read - * listener, so `feed` pushes an encoded frame into the broker. The channel - * decodes each written frame, so `responses` holds the response frames only. + * listener, so `feed` pushes the encoded envelope and its body_chunk frames into + * the broker. The channel decodes each written frame and reassembles a response + * body, so `responses` holds one reassembled response per delivered request. */ function createFakeChannelHarness(): FakeChannelHarness { - const responses: DuplexResponseFrame[] = []; + const responses: ReassembledResponse[] = []; const forwards: PendingForward[] = []; const writtenDecoder = new DuplexFrameDecoder(); + // The in-flight response reassembly, keyed by request id. + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); let dataListener: ((chunk: string) => void) | null = null; const channel: CommandManagedDuplexChannel = { write: (data) => { for (const result of writtenDecoder.push(data)) { - if (result.ok && result.frame.type === "response") { - responses.push(result.frame); + if (!result.ok) continue; + const frame = result.frame; + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + responses.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) continue; + const decoded = Buffer.from(frame.data, "base64"); + assembly.received += decoded.length; + assembly.chunks.push(decoded); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + responses.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } } } }, @@ -100,9 +150,17 @@ function createFakeChannelHarness(): FakeChannelHarness { return { channel, - feed: (frame) => { + feed: ({ frame, bodyText }) => { if (!dataListener) throw new Error("The broker did not bind the data listener."); dataListener(encodeDuplexFrame(frame)); + if (bodyText.length > 0) { + const chunks = splitBodyIntoChunkFrames( + frame.id, + Buffer.from(bodyText, "utf8"), + DUPLEX_FRAME_VERSION, + ); + for (const chunk of chunks) dataListener(encodeDuplexFrame(chunk)); + } }, responses, forwards, @@ -134,6 +192,11 @@ function controllableForward(harness: FakeChannelHarness) { }); } +/** Flush the microtasks and one macrotask, so the broker dispatches its reassembled forwards. */ +async function flush(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + /** The telemetry sink capture. It proves the broker records nothing for a refusal. */ interface TelemetryCapture { spans: DuplexTelemetrySpanRecord[]; @@ -175,11 +238,11 @@ describe("duplex bridge broker request limits", () => { expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 8, maxLifetimeRequests: 16 })).not.toThrow(); }); - it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", () => { + it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const maxInFlightRequests = 4; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, @@ -189,10 +252,13 @@ describe("duplex bridge broker request limits", () => { broker.start(); // Inject more than the limit of unique valid frames. Every forward hangs, so - // the broker holds the maximum number of in-flight forwards. + // the broker holds the maximum number of in-flight forwards. The broker gates + // the in-flight limit at the request envelope, so it refuses the excess at + // once; it dispatches each accepted forward after it reassembles the body. const total = 10; const ids = Array.from({ length: total }, (_unused, index) => `flight-${index}`); for (const id of ids) harness.feed(requestFrame(id)); + await flush(); // The broker forwarded only up to the limit. It refused the rest. expect(harness.forwards).toHaveLength(maxInFlightRequests); @@ -226,7 +292,7 @@ describe("duplex bridge broker request limits", () => { it("frees capacity after a delivered request and forwards a resent refused id (no-replay preserved)", async () => { const harness = createFakeChannelHarness(); const maxInFlightRequests = 2; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), maxInFlightRequests, @@ -234,30 +300,36 @@ describe("duplex bridge broker request limits", () => { brokers.push(broker); broker.start(); - // Saturate the in-flight limit with two hung forwards. + // Saturate the in-flight limit with two hung forwards. The broker reserves the + // in-flight slot at the request envelope, then dispatches the forward after it + // reassembles the body. harness.feed(requestFrame("keep-0")); harness.feed(requestFrame("keep-1")); + await flush(); expect(harness.forwards).toHaveLength(2); // A third unique id is refused, retryable. The broker did not forward it and // did not retain its id. harness.feed(requestFrame("resend-me")); + await flush(); expect(harness.forwards).toHaveLength(2); expect(harness.responses).toHaveLength(1); expect(JSON.parse(harness.responses[0].body).retryable).toBe(true); // Free one in-flight slot. The delivered request drains the pending record. harness.resolveForward("keep-0"); - await Promise.resolve(); + await flush(); // The gateway resends the refused id. Capacity is free now, so the broker // forwards it exactly one time. This proves the refusal did not poison the id. harness.feed(requestFrame("resend-me")); + await flush(); expect(harness.forwards.filter((forward) => forward.id === "resend-me")).toHaveLength(1); // A resend of an already-forwarded id never reaches the forward twice. const forwardedKeep1 = harness.forwards.filter((forward) => forward.id === "keep-1").length; harness.feed(requestFrame("keep-1")); + await flush(); expect(harness.forwards.filter((forward) => forward.id === "keep-1")).toHaveLength(forwardedKeep1); harness.resolveAll(); @@ -267,7 +339,7 @@ describe("duplex bridge broker request limits", () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const maxLifetimeRequests = 3; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, // Resolve every forward at once, so each dispatched id leaves the in-flight // count but stays in the retained-id set. @@ -285,7 +357,7 @@ describe("duplex bridge broker request limits", () => { const total = 9; for (let index = 0; index < total; index += 1) { harness.feed(requestFrame(`life-${index}`)); - await Promise.resolve(); + await flush(); } // The broker forwarded only up to the lifetime limit. The retained-id set @@ -304,7 +376,7 @@ describe("duplex bridge broker request limits", () => { // A resend of a refused id stays refused, so the set never grows. harness.feed(requestFrame("life-8")); - await Promise.resolve(); + await flush(); expect(harness.forwards).toHaveLength(maxLifetimeRequests); // The telemetry surface stayed on the fixed request span for the delivered @@ -320,9 +392,9 @@ describe("duplex bridge broker request limits", () => { expect(broker.state).toBe("open"); }); - it("forwards nothing and fails the channel closed under a flood of over-limit ids", () => { + it("forwards nothing and fails the channel closed under a flood of over-limit ids", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -347,9 +419,9 @@ describe("duplex bridge broker request limits", () => { expect(broker.lossRecord?.reason).toBe("protocol_failure"); }); - it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", () => { + it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -359,19 +431,21 @@ describe("duplex bridge broker request limits", () => { // An id at the maximum byte size is allowed. The broker forwards it one time. const maxId = "a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES); harness.feed(requestFrame(maxId)); + await flush(); expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); // Deliver the response, then resend the same id. The broker retained the id, // so the resend never reaches the forward a second time. harness.resolveForward(maxId); harness.feed(requestFrame(maxId)); + await flush(); expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); expect(broker.state).toBe("open"); }); it("maps a forward that resolves with the indeterminate marker to a non-retryable response and retains the id", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -384,6 +458,7 @@ describe("duplex bridge broker request limits", () => { // non-retryable body, so the gateway maps it to a non-retryable status and a // caller never repeats a possibly-committed mutation. harness.feed(requestFrame("commit-1")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "commit-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -423,7 +498,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward rejection for a mutating method to a non-retryable indeterminate response and retains the id", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -436,6 +511,7 @@ describe("duplex bridge broker request limits", () => { // forward, so a POST must return a non-retryable indeterminate response. // A retryable status would let a caller repeat a committed mutation. harness.feed(requestFrame("mutate-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -465,7 +541,7 @@ describe("duplex bridge broker request limits", () => { it("substitutes a bounded terminal response for an oversized response and keeps the channel open", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, @@ -482,6 +558,7 @@ describe("duplex bridge broker request limits", () => { // host. harness.feed(requestFrame("small-1", "POST")); harness.feed(requestFrame("big-1", "POST")); + await flush(); const smallForward = harness.forwards.find((entry) => entry.id === "small-1" && !entry.settled); if (!smallForward) throw new Error("The broker did not forward the small request."); @@ -542,30 +619,35 @@ describe("duplex bridge broker request limits", () => { it("sends a minimal terminal response when the frame bound rejects the full replacement", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); - // The bound accepts a compact request (130 bytes) and the minimal terminal - // response (106 bytes), but rejects the full replacement response (288 bytes). - // This proves the broker never drops the channel or strands the request when - // the configured bound is smaller than the full replacement response. - const broker = createDuplexBridgeBroker({ + // The bound accepts a compact request envelope (128 bytes) and the minimal + // terminal response envelope (114 bytes), but rejects the full replacement + // response envelope (193 bytes). This proves the broker never drops the channel + // or strands the request when the bound is smaller than the full replacement. + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, - maxFrameBytes: 200, + maxFrameBytes: 150, }); brokers.push(broker); broker.start(); - // A compact request frame fits the bound, so the broker forwards it. + // A compact request envelope with an empty body fits the bound, so the broker + // forwards it. harness.feed({ - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "big-1", - method: "POST", - path: "/api/issues/big-1", - query: "", - headers: {}, - body: JSON.stringify({ go: 1 }), + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "big-1", + method: "POST", + path: "/api/issues/big-1", + query: "", + headers: {}, + bodyByteCount: 0, + }, + bodyText: "", }); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -608,29 +690,34 @@ describe("duplex bridge broker request limits", () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const logs: string[] = []; - // The bound accepts a tiny request (99 bytes) but rejects even the minimal - // terminal response (102 bytes). The broker cannot send any frame for this - // request, so it must keep the channel open and log a clear local error. - const broker = createDuplexBridgeBroker({ + // The bound accepts a tiny request envelope (107 bytes) but rejects even the + // minimal terminal response envelope (110 bytes). The broker cannot send any + // frame for this request, so it must keep the channel open and log a clear + // local error. + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, - maxFrameBytes: 100, + maxFrameBytes: 109, logger: (message) => logs.push(message), }); brokers.push(broker); broker.start(); harness.feed({ - version: DUPLEX_FRAME_VERSION, - type: "request", - id: "x", - method: "GET", - path: "/", - query: "", - headers: {}, - body: "", + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "x", + method: "GET", + path: "/", + query: "", + headers: {}, + bodyByteCount: 0, + }, + bodyText: "", }); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "x" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -657,7 +744,7 @@ describe("duplex bridge broker request limits", () => { it("completes a request within the bound and delivers the real response", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, @@ -667,6 +754,7 @@ describe("duplex bridge broker request limits", () => { broker.start(); harness.feed(requestFrame("ok-1", "POST")); + await flush(); harness.resolveForward("ok-1", JSON.stringify({ ok: true })); await new Promise((resolve) => setTimeout(resolve, 0)); @@ -690,7 +778,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward rejection for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -701,6 +789,7 @@ describe("duplex bridge broker request limits", () => { // mutation. A forward rejection for a GET stays retryable: the broker // returns a 502 with the completed outcome, so the gateway passes it through. harness.feed(requestFrame("read-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -719,7 +808,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward-budget timeout for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // A tiny forward budget aborts the controller before the manual rejection. @@ -732,6 +821,7 @@ describe("duplex bridge broker request limits", () => { // stay retryable: the broker returns a 504 with the completed outcome and no // indeterminate marker, so the gateway does not map it to a terminal 409. harness.feed(requestFrame("read-timeout-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-timeout-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -755,7 +845,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward-budget timeout for a mutating method to a non-retryable indeterminate response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), budgets: { forwardTimeoutMs: 5 }, @@ -767,6 +857,7 @@ describe("duplex bridge broker request limits", () => { // must keep the terminal contract: a 504 with the indeterminate outcome, so // the gateway maps it to a non-retryable 409 and no caller double-applies it. harness.feed(requestFrame("mutate-timeout-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-timeout-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -789,7 +880,7 @@ describe("duplex bridge broker request limits", () => { it("maps a response-budget backstop for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // The forward budget aborts the call, but the forward promise stays @@ -806,6 +897,7 @@ describe("duplex bridge broker request limits", () => { // retryable: a 504 with the completed outcome and no indeterminate marker, // so the gateway does not map it to a terminal 409. harness.feed(requestFrame("read-stall-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-stall-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -826,7 +918,7 @@ describe("duplex bridge broker request limits", () => { it("maps a response-budget backstop for a mutating method to a non-retryable indeterminate response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 }, @@ -839,6 +931,7 @@ describe("duplex bridge broker request limits", () => { // contract: a 504 with the indeterminate outcome, so the gateway maps it to a // non-retryable 409 and no caller double-applies it. harness.feed(requestFrame("mutate-stall-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-stall-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -897,10 +990,10 @@ describe("duplex bridge broker exit taxonomy", () => { } }); - it("records a transport close as a distinct loss, not a process exit", () => { + it("records a transport close as a distinct loss, not a process exit", async () => { const { telemetry, capture } = createTelemetryCapture(); const harness = createExitChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: () => Promise.resolve({ status: 200, headers: {}, body: "" }), @@ -920,10 +1013,10 @@ describe("duplex bridge broker exit taxonomy", () => { expect(lossCounter?.dimensions.loss_reason).toBe("transport_closed"); }); - it("records a numeric exit as a process exit", () => { + it("records a numeric exit as a process exit", async () => { const { telemetry, capture } = createTelemetryCapture(); const harness = createExitChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: () => Promise.resolve({ status: 200, headers: {}, body: "" }), diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts index b7a04021a15..1ae50d92bb6 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.ts @@ -48,17 +48,26 @@ import { type ReservationToken, } from "./duplex-aggregate-byte-ledger.js"; import { + DUPLEX_BODY_CHUNK_RAW_BYTES, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, DUPLEX_FRAME_VERSION, DuplexFrameDecoder, encodeDuplexFrame, encodeDuplexFrameChecked, + type DuplexBodyChunkFrame, type DuplexFrame, type DuplexRequestFrame, type DuplexResponseFrame, type DuplexResponseOutcome, } from "./duplex-frame-codec.js"; +import { + DuplexBodyError, + DuplexBodyReceiver, + splitBodyIntoChunkFrames, + type DuplexBodyReceiverConfig, + type ReassembledBody, +} from "./duplex-body-spool.js"; import type { DuplexLossReason, DuplexOutcomeValue, @@ -195,12 +204,15 @@ export interface DuplexBrokerForwardResult { /** * The forward handler the broker calls for each request. The handler applies the * real token and the run attribution, then forwards the request on the existing - * API path. The broker aborts `options.signal` when the forward budget ends or a - * loss happens, so a handler that threads the signal into its work stops early. + * API path. The broker reassembles the request body from the `body_chunk` frames + * before it calls the handler, so the handler streams the body from + * `options.body`. The broker aborts `options.signal` when the forward budget ends + * or a loss happens, so a handler that threads the signal into its work stops + * early. */ export type DuplexBrokerForwardHandler = ( request: DuplexRequestFrame, - options: { signal: AbortSignal }, + options: { signal: AbortSignal; body: ReassembledBody }, ) => Promise; /** One request record. The broker captures the dispatch-start point for metrics only. */ @@ -249,6 +261,14 @@ export interface DuplexBrokerOptions { * {@link DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS}. */ maxLifetimeRequests?: number; + /** + * The config for the receive-side request-body reassembler. It sets the spill + * threshold, the fixed raw chunk size, and the per-channel spill caps. The + * broker creates one reassembler for the channel. The config stays injectable, + * so a test lowers the spill threshold and the caps to exercise the spill path + * and the fail-closed cap behavior without a large body. + */ + bodyReceiverConfig?: DuplexBodyReceiverConfig; /** The clock the broker reads for the metric timestamps. The default is `Date.now`. */ now?: () => number; /** The metrics sink for the per-request dispatch record. */ @@ -381,10 +401,24 @@ export function isSafeBridgeMethod(method: string): boolean { interface PendingRequest { controller: AbortController; responded: boolean; - forwardTimer: ReturnType; + /** + * The forward-budget timer. It is `null` while the broker reassembles the + * request body, and the broker sets it when it starts the forward. So the + * forward budget bounds the forward call, not the reassembly. + */ + forwardTimer: ReturnType | null; + /** + * The response-budget backstop timer. The broker starts it at the request + * envelope, so it bounds the whole request: the body reassembly plus the + * forward. It answers a request that never completes its reassembly or forward. + */ responseTimer: ReturnType; /** The point the broker started to dispatch the request. It sets the span latency. */ dispatchStartMs: number; + /** The request method. The response backstop reads it to classify the safe-method retry. */ + method: string; + /** The reassembled request body, set once reassembly completes. The broker disposes it on settle. */ + reassembled: ReassembledBody | null; /** True once the forward promise settled. The finally owner sets it one time. */ forwardSettled: boolean; /** @@ -409,10 +443,14 @@ interface OrphanedForward { } /** - * Create the host duplex bridge broker. The factory asserts the budget order and - * returns a handle. Call `start` to wire the channel and open the broker. + * Create the host duplex bridge broker. The factory asserts the budget order, + * creates the receive-side request-body reassembler, and returns a handle. It is + * asynchronous because the reassembler owns a spill directory it creates with + * `mkdtemp`. Call `start` to wire the channel and open the broker. */ -export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBridgeBroker { +export async function createDuplexBridgeBroker( + options: DuplexBrokerOptions, +): Promise { const budgets: DuplexBrokerBudgets = { ...DEFAULT_DUPLEX_BROKER_BUDGETS, ...options.budgets, @@ -452,21 +490,50 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr if (ledger && token) ledger.release(token); }; // The byte size of the normalized request payload the broker retains across the - // forward. It counts the exact retained scalar and header bytes, so the charge - // matches the retained bytes and never a parsed object graph. + // forward. It counts the exact retained scalar and header bytes plus the raw body + // byte count, so the charge matches the reassembled body the broker holds and + // never a parsed object graph. The body rides `body_chunk` frames, so the + // envelope carries only `bodyByteCount`; that count is the retained body size. const requestPayloadBytes = (frame: DuplexRequestFrame): number => { let bytes = Buffer.byteLength(frame.id, "utf8") + Buffer.byteLength(frame.method, "utf8") + Buffer.byteLength(frame.path, "utf8") + Buffer.byteLength(frame.query, "utf8") + - Buffer.byteLength(frame.body, "utf8"); + frame.bodyByteCount; for (const [key, value] of Object.entries(frame.headers)) { bytes += Buffer.byteLength(key, "utf8") + Buffer.byteLength(value, "utf8"); } return bytes; }; + // The receive-side request-body reassembler. It reassembles each request body + // from the `body_chunk` frames, on the memory path at or below the spill + // threshold and on the spill path above it, so the broker never holds a whole + // large request body in memory. It owns a per-channel spill directory and the + // spill caps. + const receiver = await DuplexBodyReceiver.create(options.bodyReceiverConfig ?? {}); + let receiverFinalized = false; + const finalizeReceiver = (): Promise => { + // Remove the spill directory and every in-flight body once. The broker calls + // it on every terminal path: a loss, a normal teardown, and an orderly close. + if (receiverFinalized) return Promise.resolve(); + receiverFinalized = true; + return receiver.destroy().catch(() => undefined); + }; + // The fixed raw slice size for the response `body_chunk` frames. It matches the + // reassembler config, so a test that lowers the slice size splits both a request + // body and a response body the same way. + const rawChunkBytes = options.bodyReceiverConfig?.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; + // The ids the broker is reassembling right now. A `body_chunk` for an id in this + // set routes to the reassembler. The broker removes an id when the body settles. + const reassembling = new Set(); + // The ids the broker refused at the envelope, with the raw bytes it still must + // drain. The sender emits the `body_chunk` frames for a refused request before + // it learns of the refusal, so the broker drains and drops those chunks instead + // of treating them as a body_chunk with no envelope. + const draining = new Map(); + let state: DuplexBrokerState = "opening"; let stopped = false; let started = false; @@ -544,21 +611,33 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const clearPending = (): void => { for (const [id, entry] of pending) { - clearTimeout(entry.forwardTimer); + if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); clearTimeout(entry.responseTimer); entry.controller.abort(new Error("Duplex broker stopped.")); - // A forward that has not settled still owns its request tokens. Transfer it - // to the orphan registry, so its tokens stay charged until the forward - // finally releases them. The abort only asks the forward to stop; it never - // releases a token by itself. - if (!entry.forwardSettled) { + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + if (entry.forwardSettled) continue; + if (entry.forwardTimer !== null) { + // A live forward promise still owns its request tokens. Transfer it to the + // orphan registry, so its tokens stay charged until the forward finally + // releases them. The abort only asks the forward to stop; it never releases + // a token by itself. orphanedForwards.set(id, { controller: entry.controller, releaseForwardTokens: entry.releaseForwardTokens, }); + } else { + // The request is still reassembling, so no forward promise will settle and + // release its tokens. Release the request tokens now. The reassembler + // teardown below rejects the in-flight body and unlinks any spill file. + entry.releaseForwardTokens(); } } pending.clear(); + reassembling.clear(); + draining.clear(); // Ask every already-orphaned forward to stop as well. The forward-promise // finally owner releases each orphan token when the forward settles. for (const orphan of orphanedForwards.values()) { @@ -582,6 +661,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr clearPending(); releaseSeenRequestIdTokens(); decoder.dispose(); + void finalizeReceiver(); if (state !== "closing") setState("closed"); return; } @@ -607,6 +687,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr clearPending(); releaseSeenRequestIdTokens(); decoder.dispose(); + void finalizeReceiver(); lossRecord = { reason, message, atMs: now() }; setState("lost"); // Log the internal reason only. The broker never writes the raw provider @@ -626,31 +707,57 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } }; - // The result of one bounded send. `sent` means the frame went out; `too_large` - // means the encoded frame exceeds the bound and the broker wrote nothing; - // `lost` means the write failed and the broker recorded the channel loss. + // The result of one bounded send. `sent` means every frame went out; `too_large` + // means the response envelope exceeds the bound and the broker wrote nothing; + // `lost` means a write failed and the broker recorded the channel loss. type SendResult = "sent" | "too_large" | "lost"; - const trySendFrame = (frame: DuplexFrame): SendResult => { - // Enforce the frame size bound on every broker write. Report a size rejection - // as `too_large` without a channel loss; a broker-built frame over the bound - // is a defect, not a transport failure. A caller decides how to answer the - // request, so no size rejection ever drops the channel. - const encoded = encodeDuplexFrameChecked(frame, maxFrameBytes); - if (!encoded.ok) return "too_large"; - return writeLine(encoded.line) ? "sent" : "lost"; - }; - - const writeFrame = (frame: DuplexFrame): boolean => { - // A control frame and the bounded terminal responses are always small, so the - // size guard is a no-op for them. Drop an oversized frame here without a - // channel loss and report the drop to the caller. - const result = trySendFrame(frame); - if (result === "too_large") { - options.logger?.(`Duplex broker dropped an oversized ${frame.type} frame.`); - return false; + // Send one full response as a sequence of frames: one envelope frame that + // carries `bodyByteCount`, then the `body_chunk` frames that carry the body. The + // envelope is always small. Each `body_chunk` carries one fixed raw slice, so + // each encoded chunk stays under the frame bound by construction. The result + // reports `too_large` when the envelope itself exceeds the bound and the broker + // wrote nothing, `lost` when a write failed and the broker recorded the channel + // loss, and `sent` when every frame went out. + const sendResponseFrames = ( + id: string, + status: number, + headers: Record, + bodyText: string, + outcome: DuplexResponseOutcome, + ): SendResult => { + const bodyBuffer = Buffer.from(bodyText, "utf8"); + const envelope: DuplexResponseFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status, + headers, + bodyByteCount: bodyBuffer.length, + outcome, + }; + // Guard the envelope against the frame bound. The envelope holds no body, so a + // real bound rejects it only under an extreme small test bound. Report the + // rejection without a channel loss, so the caller decides how to answer. + const encodedEnvelope = encodeDuplexFrameChecked(envelope, maxFrameBytes); + if (!encodedEnvelope.ok) return "too_large"; + const chunkFrames = splitBodyIntoChunkFrames(id, bodyBuffer, DUPLEX_FRAME_VERSION, rawChunkBytes); + if (!writeLine(encodedEnvelope.line)) return "lost"; + for (const chunk of chunkFrames) { + const encodedChunk = encodeDuplexFrameChecked(chunk, maxFrameBytes); + if (!encodedChunk.ok) { + // A single fixed-size slice never exceeds the bound in a real configuration. + // A test bound below one chunk can reject it. The envelope already went out + // with the true `bodyByteCount`, so the broker cannot complete the body + // within the bound. Log the drop and stop; the gateway ends the request on + // its wait budget. Report `sent`, because the broker already committed the + // envelope, so no caller resends a bounded replacement over the same id. + options.logger?.("Duplex broker dropped an oversized body_chunk frame."); + return "sent"; + } + if (!writeLine(encodedChunk.line)) return "lost"; } - return result === "sent"; + return "sent"; }; const sendTerminalIndeterminate = (id: string): void => { @@ -658,42 +765,32 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // request reached the host and may have changed state, so the response is // non-retryable and carries the `indeterminate` outcome. The broker tries the // full replacement first. When the frame bound rejects even the full - // replacement, the broker sends a minimal replacement that carries only the - // `indeterminate` outcome, so the gateway still ends the request and never - // waits for its full wait budget. When the bound rejects even the minimal - // replacement, the broker logs a clear local error and keeps the channel - // open; it records no channel loss. - const fullReplacement: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: 502, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "upstream response too large to deliver", - outcome: "indeterminate", - retryable: false, - }), + // replacement envelope, the broker sends a minimal replacement that carries + // only the `indeterminate` outcome, so the gateway still ends the request and + // never waits for its full wait budget. When the bound rejects even the + // minimal replacement envelope, the broker logs a clear local error and keeps + // the channel open; it records no channel loss. + const fullBody = JSON.stringify({ + error: "upstream response too large to deliver", outcome: "indeterminate", - }; - if (trySendFrame(fullReplacement) !== "too_large") return; - // The bound rejects the full replacement. Send a minimal terminal response - // with empty headers and an empty body. The `indeterminate` outcome still - // rides the frame, so the gateway maps the request to a terminal 409. - const minimalReplacement: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: 502, - headers: {}, - body: "", - outcome: "indeterminate", - }; - if (trySendFrame(minimalReplacement) !== "too_large") return; - // The bound rejects even the minimal terminal response. The broker cannot + retryable: false, + }); + if ( + sendResponseFrames( + id, + 502, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "indeterminate" }, + fullBody, + "indeterminate", + ) !== "too_large" + ) { + return; + } + // The bound rejects the full replacement envelope. Send a minimal terminal + // response with empty headers and an empty body. The `indeterminate` outcome + // still rides the envelope, so the gateway maps the request to a terminal 409. + if (sendResponseFrames(id, 502, {}, "", "indeterminate") !== "too_large") return; + // The bound rejects even the minimal terminal envelope. The broker cannot // deliver any frame for this request within the bound. Log a clear local // error and keep the channel open for every other request. The gateway ends // its own outstanding request on its wait budget. @@ -711,40 +808,19 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const entry = pending.get(id); if (!entry || entry.responded) return; entry.responded = true; - clearTimeout(entry.forwardTimer); - clearTimeout(entry.responseTimer); - pending.delete(id); - // The response timer, an abort, or a broker loss can answer the gateway before - // the forward promise settles. When the forward has not settled, transfer it to - // the orphan registry so its request tokens stay charged until the forward - // finally releases them. A settled forward already released or will release - // through its finally owner, so it needs no orphan record. - if (!entry.forwardSettled) { - orphanedForwards.set(id, { - controller: entry.controller, - releaseForwardTokens: entry.releaseForwardTokens, - }); - } + settlePendingBookkeeping(id, entry); // Do not write on a lost or closed channel. The gateway answers its own // outstanding request on loss, so a late write would go to a dead channel. if (state !== "open") return; - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: result.status, - headers: result.headers ?? {}, - body: result.body ?? "", - outcome, - }; - const encoded = encodeDuplexFrameChecked(frame, maxFrameBytes); - if (!encoded.ok) { - // The host produced a result, but the response frame exceeds the size - // bound. Do not write the oversized frame and do not record a channel loss. - // The request reached the host and may have changed state, so the caller - // must not retry a possible mutation. Send a bounded, non-retryable terminal - // response marked `indeterminate` in its place. Record the request outcome - // as an error, never a loss. The channel stays open for every other request. + const bodyText = result.body ?? ""; + const bodyByteCount = Buffer.byteLength(bodyText, "utf8"); + // The response body rides `body_chunk` frames, so a large body no longer makes + // one frame too large. The gateway reassembles a response body in memory, + // though, so a body over the frame bound would force the gateway to hold an + // oversized body in memory. Treat a response body over the frame bound like the + // former oversized case: send a bounded, non-retryable indeterminate terminal + // response, and record the request outcome as an error, never a loss. + if (bodyByteCount > maxFrameBytes) { options.telemetry?.recordRequest({ latencyMs: now() - entry.dispatchStartMs, outcome: "error", @@ -753,12 +829,14 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr return; } // Record the request span for the delivered request. The span carries the - // latency and the outcome only; no route, query, body, or token rides it. + // latency and the outcome only; no route, query, body, or token rides it. The + // broker records the span before the writes, so the outcome is recorded even if + // a later chunk write records a channel loss. options.telemetry?.recordRequest({ latencyMs: now() - entry.dispatchStartMs, outcome: telemetryOutcome, }); - writeLine(encoded.line); + sendResponseFrames(id, result.status, result.headers ?? {}, bodyText, outcome); }; const respondSaturated = (id: string, retryable: boolean): void => { @@ -768,23 +846,61 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // it holds only the fixed error shape. The `unavailable` outcome tells the // gateway this is not a delivered host response, so it never counts as one. if (state !== "open") return; - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", + sendResponseFrames( id, - status: 503, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "unavailable", - }, - body: JSON.stringify({ + 503, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, + JSON.stringify({ error: "Duplex broker capacity limit reached.", outcome: "unavailable", retryable, }), - outcome: "unavailable", - }; - writeFrame(frame); + "unavailable", + ); + }; + + // Release the timers and route the reassembly and ledger resources for one + // request the broker just answered. The broker calls it exactly once per + // request, from the response path. It clears both timers and drops the pending + // record, then it settles the resources by the request phase: + // - A live forward promise (the response backstop answered while the forward + // ran) keeps its request tokens and its reassembled body until its finally + // owner releases and disposes them, so the orphan registry holds it. + // - A request still reassembling (the backstop answered before the forward + // started) has no forward promise to settle, so the broker releases its + // tokens and disposes the in-flight body now. + // - A settled forward already released and disposed through its finally owner. + const settlePendingBookkeeping = (id: string, entry: PendingRequest): void => { + if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); + clearTimeout(entry.responseTimer); + pending.delete(id); + if (entry.forwardSettled) return; + if (entry.forwardTimer !== null) { + // A live forward still owns its request tokens and streams the reassembled + // body. Keep both charged until the forward finally owner settles them. + orphanedForwards.set(id, { + controller: entry.controller, + releaseForwardTokens: entry.releaseForwardTokens, + }); + return; + } + // The request is still reassembling, so no forward promise will settle. Release + // the request tokens and dispose the in-flight body, so no spill file, arena + // reservation, or ledger token lingers. + entry.releaseForwardTokens(); + if (reassembling.delete(id)) void receiver.disposeBody(id); + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + }; + + // Mark a refused or duplicate request id for chunk draining. The sender emits + // the `body_chunk` frames for the request before it learns of the refusal, so + // the broker records the raw byte count it must drain and drop. A request with + // no body needs no draining, so the broker records only a non-zero count. + const markDrain = (frame: DuplexRequestFrame): void => { + if (frame.bodyByteCount > 0) draining.set(frame.id, frame.bodyByteCount); }; const respondAggregateExceeded = (id: string): void => { @@ -796,23 +912,17 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // host response. The refusal is retryable, because the broker did not retain // the id: the aggregate pressure can ease, and a resend can then get through. if (state !== "open") return; - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", + sendResponseFrames( id, - status: 503, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "unavailable", - }, - body: JSON.stringify({ + 503, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, + JSON.stringify({ error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, outcome: "unavailable", retryable: true, }), - outcome: "unavailable", - }; - writeFrame(frame); + "unavailable", + ); }; const dispatch = (frame: DuplexRequestFrame): void => { @@ -825,13 +935,19 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // forwards. It answers with the bounded terminal refusal, which carries no // route, query, body, or token, and records no telemetry. The refusal is not // retryable, because a resend of the same over-limit id never gets past this - // bound. + // bound. It drains the following chunks of the refused body. if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { respondSaturated(frame.id, false); + markDrain(frame); + return; + } + // Forward one id one time. A repeated id never reaches the API twice. The + // broker already answered the first request, so it answers no second time; it + // drains the resent body chunks and drops them. + if (seenRequestIds.has(frame.id)) { + markDrain(frame); return; } - // Forward one id one time. A repeated id never reaches the API twice. - if (seenRequestIds.has(frame.id)) return; // Bound the retained request-id memory. The broker keeps one id per distinct // dispatched request for the no-replay guarantee, so the set can only grow. // Once the broker reaches the lifetime limit, it refuses each new distinct id @@ -840,6 +956,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // the set never grows past the limit. if (seenRequestIds.size >= limits.maxLifetimeRequests) { respondSaturated(frame.id, false); + markDrain(frame); return; } // Bound the in-flight request count. Once the broker holds the maximum number @@ -847,9 +964,11 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // it. The broker does not add the id to the seen set, so the gateway can // resend the request after an in-flight request completes. The refusal is // retryable for that reason. This check bounds the controllers, the timers, - // and the concurrent forwards a provider can force. + // the concurrent reassemblies, and the concurrent forwards a provider can + // force. if (pending.size >= limits.maxInFlightRequests) { respondSaturated(frame.id, true); + markDrain(frame); return; } // Reserve the exact retained bytes against the aggregate ledger before the @@ -868,12 +987,14 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr requestFrameToken = ledger.reserve("request_frame", rawFrameBytes); if (!requestFrameToken) { respondAggregateExceeded(frame.id); + markDrain(frame); return; } requestPayloadToken = ledger.reserve("request_payload", requestPayloadBytes(frame)); if (!requestPayloadToken) { ledger.release(requestFrameToken); respondAggregateExceeded(frame.id); + markDrain(frame); return; } const seenEntryBytes = @@ -883,6 +1004,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr ledger.release(requestFrameToken); ledger.release(requestPayloadToken); respondAggregateExceeded(frame.id); + markDrain(frame); return; } } @@ -910,83 +1032,123 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr options.onRequestRecord?.(record); const controller = new AbortController(); - const forwardTimer = setTimeout(() => { - controller.abort(new Error("Duplex broker forward budget exceeded.")); - }, budgets.forwardTimeoutMs); - const responseTimer = setTimeout(() => { - // Response-budget backstop. The forward rejection normally answers first, - // well before this deadline. This backstop answers a request whose forward - // rejection handling itself stalls, so the request never strands. One stall - // path is a response whose headers arrive but whose body reader stays - // pending through the budget, so the forward promise never settles. - if (isSafeBridgeMethod(frame.method)) { - // A safe method never changes host state, so a stalled body reader - // cannot leave a mutation half-applied. Keep the request retryable: - // return a 504 with the completed outcome and no indeterminate marker, - // so the gateway passes it through as a retryable status and never maps - // it to a terminal 409. - respond( - frame.id, - { - status: 504, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - retryable: true, - }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, and the broker cannot prove the host - // applied no mutation once the response budget passes. Return a - // non-retryable 504 and mark the outcome indeterminate, so the gateway - // maps it to a terminal 409 and no caller double-applies the mutation. - respond( - frame.id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - }, budgets.responseBudgetMs); - forwardTimer.unref?.(); + // Start the response-budget backstop at the request envelope, so it bounds the + // whole request: the body reassembly plus the forward. The forward-budget timer + // starts later, when the broker starts the forward, so it bounds the forward + // call alone. + const responseTimer = setTimeout(() => respondBackstop(frame.id), budgets.responseBudgetMs); responseTimer.unref?.(); - pending.set(frame.id, { + const entry: PendingRequest = { controller, responded: false, - forwardTimer, + forwardTimer: null, responseTimer, dispatchStartMs: record.dispatchStartMs, + method: frame.method, + reassembled: null, forwardSettled: false, releaseForwardTokens, - }); - - // The forward promise has one cleanup owner. The two settle handlers mark the - // forward settled and answer the gateway. The `finally` then releases the - // request tokens exactly one time and removes the request from both the pending - // map and the orphan map. The forward resolves only after the forward and its - // response-body reader both settle, so this owner waits for both. - const markForwardSettled = (): void => { - // Mark the pending entry settled before `respond` runs, so `respond` never - // moves a just-settled forward to the orphan registry. An already-orphaned - // forward needs no flag; its finally owner releases and drains it. - const entry = pending.get(frame.id); - if (entry) entry.forwardSettled = true; }; - forwardRequest(frame, { signal: controller.signal }) + pending.set(frame.id, entry); + // Route the following `body_chunk` frames for this id to the reassembler. + reassembling.add(frame.id); + // Reassemble the request body from the `body_chunk` frames, then forward it. + // The reassembler chooses the memory path or the spill path from the envelope + // `bodyByteCount`, so the broker never holds a whole large request body in + // memory. A reassembly error is a terminal protocol failure. + receiver.begin(frame.id, frame.bodyByteCount).then( + (body) => onBodyReady(frame, entry, body), + (error) => onBodyError(frame.id, entry, error), + ); + }; + + const respondBackstop = (id: string): void => { + // Response-budget backstop. The forward rejection normally answers first, well + // before this deadline. This backstop answers a request whose body never + // completes its reassembly, or whose forward rejection handling itself stalls, + // so the request never strands. One stall path is a response whose headers + // arrive but whose body reader stays pending through the budget, so the forward + // promise never settles. + const entry = pending.get(id); + if (!entry || entry.responded) return; + if (isSafeBridgeMethod(entry.method)) { + // A safe method never changes host state, so a stalled body reader cannot + // leave a mutation half-applied. Keep the request retryable: return a 504 + // with the completed outcome and no indeterminate marker, so the gateway + // passes it through as a retryable status and never maps it to a terminal 409. + respond( + id, + { + status: 504, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + error: "Duplex broker response budget exceeded.", + retryable: true, + }), + }, + "completed", + "error", + ); + return; + } + // The method may mutate host state, and the broker cannot prove the host + // applied no mutation once the response budget passes. Return a non-retryable + // 504 and mark the outcome indeterminate, so the gateway maps it to a terminal + // 409 and no caller double-applies the mutation. + respond( + id, + { + status: 504, + headers: { + "content-type": "application/json", + "x-paperclip-bridge-outcome": "indeterminate", + }, + body: JSON.stringify({ + error: "Duplex broker response budget exceeded.", + outcome: "indeterminate", + retryable: false, + }), + }, + "indeterminate", + "error", + ); + }; + + // The request body reassembled. Start the forward now, so the forward budget + // bounds the forward call alone. The broker passes the reassembled body to the + // forward handler, so the handler streams it to the host API path. + const onBodyReady = ( + frame: DuplexRequestFrame, + entry: PendingRequest, + body: ReassembledBody, + ): void => { + reassembling.delete(frame.id); + if (stopped || state !== "open" || entry.responded) { + // The channel died, or the response backstop already answered, so no forward + // runs. Dispose the body and release the request tokens, so no spill file, + // arena reservation, or ledger token lingers, and drop the request from the + // pending and orphan maps. + void body.dispose(); + pending.delete(frame.id); + orphanedForwards.delete(frame.id); + entry.releaseForwardTokens(); + return; + } + entry.reassembled = body; + const forwardTimer = setTimeout(() => { + entry.controller.abort(new Error("Duplex broker forward budget exceeded.")); + }, budgets.forwardTimeoutMs); + forwardTimer.unref?.(); + entry.forwardTimer = forwardTimer; + + // The forward promise has one cleanup owner. Both settle handlers mark the + // forward settled and answer the gateway. The `finally` then releases the + // request tokens exactly one time, disposes the reassembled body, and removes + // the request from the pending map and the orphan map. + const markForwardSettled = (): void => { + entry.forwardSettled = true; + }; + forwardRequest(frame, { signal: entry.controller.signal, body }) .then( (result) => { markForwardSettled(); @@ -1004,20 +1166,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr }, (error) => { markForwardSettled(); - return handleForwardRejection(error); - }, - ) - .finally(() => { - pending.delete(frame.id); - orphanedForwards.delete(frame.id); - releaseForwardTokens(); - }); - - // The forward rejection handler. It classifies the rejection by method safety - // and answers the gateway. The `markForwardSettled` call above runs first, so - // the response never re-orphans a settled forward. - function handleForwardRejection(error: unknown): void { - if (controller.signal.aborted) { + if (entry.controller.signal.aborted) { // The forward budget aborted the call. A safe method never changes // host state, so a forward timeout stays retryable for it. Return a // 504 with the completed outcome and no indeterminate marker, so the @@ -1103,7 +1252,61 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr "indeterminate", "error", ); + }, + ) + .finally(() => { + // The single forward cleanup owner. Release the request tokens, dispose the + // reassembled body, and drop the request from the pending and orphan maps, + // each exactly one time. + pending.delete(frame.id); + orphanedForwards.delete(frame.id); + entry.releaseForwardTokens(); + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + }); + }; + + // The request body reassembly failed. A malformed chunk, a size mismatch, an + // overrun, or a truncation is a terminal protocol failure, so the broker fails + // the channel closed. A `channel_closed` error is the broker's own teardown of an + // in-flight body, so it is not a fresh loss. + const onBodyError = (id: string, entry: PendingRequest, error: unknown): void => { + reassembling.delete(id); + // The forward never ran, so release its request tokens and drop the request + // from the pending and orphan maps. The release is idempotent, so a teardown + // that already released the tokens keeps the count correct. + pending.delete(id); + orphanedForwards.delete(id); + entry.releaseForwardTokens(); + if (stopped) return; + if (error instanceof DuplexBodyError && error.code === "channel_closed") return; + recordLoss("protocol_failure", errorMessage(error)); + }; + + // Route one `body_chunk` frame. A chunk for an id under reassembly goes to the + // reassembler, which validates it and appends it. A chunk for a refused or + // duplicate id drains against its recorded byte count and drops. A chunk with no + // matching envelope is a terminal protocol failure. + const handleBodyChunk = (frame: DuplexBodyChunkFrame): void => { + if (reassembling.has(frame.id)) { + const result = receiver.pushChunk(frame); + if (!result.ok) recordLoss("protocol_failure", result.error.message); + return; } + const remaining = draining.get(frame.id); + if (remaining !== undefined) { + // Drain and drop a chunk of a refused body. The broker does not validate a + // drained chunk; it only tracks the byte progress, so it stops draining once + // the refused body ends. A further chunk after that is a chunk with no + // envelope. + const next = remaining - Buffer.byteLength(frame.data, "base64"); + if (next > 0) draining.set(frame.id, next); + else draining.delete(frame.id); + return; + } + recordLoss("protocol_failure", "body_chunk arrived with no matching envelope"); }; const handleFrame = (frame: DuplexFrame): void => { @@ -1111,6 +1314,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr case "request": dispatch(frame); return; + case "body_chunk": + handleBodyChunk(frame); + return; case "close": // The gateway asked for an orderly close. The gateway sends this frame on // the agent's orderly completion, so mark the completion on the ordered @@ -1201,6 +1407,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr if (closeTimer !== undefined) clearTimeout(closeTimer); recordLoss("close_timeout", errorMessage(error)); } + // Remove the spill directory and every in-flight body. The broker owns the + // reassembler, so the orderly close cleans it up. + await finalizeReceiver(); })(); return closePromise; }; diff --git a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts index a4e2942c974..91485d79080 100644 --- a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts @@ -4,6 +4,7 @@ import net from "node:net"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { Readable } from "node:stream"; import { StringDecoder } from "node:string_decoder"; import { afterEach, describe, expect, it } from "vitest"; @@ -14,6 +15,9 @@ import { } from "./sandbox-callback-bridge.js"; import { DuplexFrameDecoder, + DUPLEX_BODY_CHUNK_RAW_BYTES, + DUPLEX_FRAME_VERSION, + encodeDuplexFrame, type DuplexFrame, type DuplexReadyFrame, type DuplexRequestFrame, @@ -23,6 +27,7 @@ import { type DuplexBridgeBroker, type DuplexBrokerForwardResult, } from "./duplex-bridge-broker.js"; +import type { ReassembledBody } from "./duplex-body-spool.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; /** @@ -299,7 +304,7 @@ async function createHarness(options: HarnessOptions = {}): Promise => { forwardedRequests.push(request); // Keep the real route allowlist on the forward seam. The broker forwards @@ -334,12 +339,16 @@ async function createHarness(options: HarnessOptions = {}): Promise; + forwardInit.duplex = "half"; + } + const response = await fetch(target, forwardInit); const body = await response.text(); const outHeaders: Record = {}; response.headers.forEach((value, key) => { @@ -349,7 +358,7 @@ async function createHarness(options: HarnessOptions = {}): Promise undefined, @@ -456,7 +465,7 @@ describe("duplex bridge local end-to-end harness", () => { const ready = harness.observedFrames.find( (frame) => frame.type === "ready", ) as DuplexReadyFrame; - expect(ready.version).toBe(1); + expect(ready.version).toBe(2); expect(ready.nonce).toBe(harness.nonce); // READY is liveness only. It carries no address data. expect((ready as unknown as Record).address).toBeUndefined(); @@ -643,4 +652,90 @@ describe("duplex bridge local end-to-end harness", () => { expect(harness.api.listening()).toBe(false); expect(harness.api.openSocketCount()).toBe(0); }, 20000); + + it("fails a request with a local 502 when the host sends a malformed response chunk", async () => { + const RAW = DUPLEX_BODY_CHUNK_RAW_BYTES; + // Each case declares one malformed body_chunk a broken or hostile host could + // send. The gateway must reject it at once with a local 502. It must not grow + // its response reassembly buffer until the response timeout. `bodyByteCount` + // sets the declared body size on the response envelope. `data` is the base64 + // payload of the one injected chunk. + const cases: Array<{ name: string; bodyByteCount: number; data: string; error: string }> = [ + { name: "an empty chunk", bodyByteCount: RAW, data: "", error: "duplex response body_chunk is empty" }, + { + name: "an undersized non-final chunk", + bodyByteCount: RAW + 8, + data: Buffer.alloc(4, 1).toString("base64"), + error: "duplex response body_chunk has the wrong size", + }, + { + name: "an oversized chunk", + bodyByteCount: RAW + 8, + data: Buffer.alloc(RAW + 4, 1).toString("base64"), + error: "duplex response body_chunk has the wrong size", + }, + { + name: "an overrun past the declared size", + bodyByteCount: 10, + data: Buffer.alloc(200, 1).toString("base64"), + error: "duplex response body overruns the declared size", + }, + { + name: "a non-canonical base64 chunk", + bodyByteCount: 10, + data: "AB==", + error: "duplex response body_chunk is not canonical base64", + }, + ]; + + const harness = await createHarness(); + harnesses.push(harness); + await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); + + // Hold the broker forward open, so only the injected frames answer each + // request. The gateway reassembles the response body itself, so a raw + // malformed chunk on its stdin drives the reject path under test. + harness.setForwardMode("hang"); + + for (const testCase of cases) { + const forwardedBefore = harness.forwardedRequests.length; + const pendingFetch = fetch(`${harness.baseUrl}/api/agents/me`, { + headers: { authorization: `Bearer ${harness.bridgeToken}` }, + }); + void pendingFetch.catch(() => undefined); + await harness.waitFor( + () => harness.forwardedRequests.length > forwardedBefore, + `the broker to receive the forwarded request for ${testCase.name}`, + ); + const id = harness.forwardedRequests[harness.forwardedRequests.length - 1].id; + + // Inject the response envelope, then the one malformed body_chunk, straight + // to the gateway stdin. This bypasses the broker response encoder, so the + // gateway sees the exact bytes a broken or hostile host could send. + harness.child.stdin.write( + encodeDuplexFrame({ + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: 200, + headers: {}, + bodyByteCount: testCase.bodyByteCount, + outcome: "completed", + }), + ); + harness.child.stdin.write( + encodeDuplexFrame({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id, + seq: 0, + data: testCase.data, + }), + ); + + const response = await pendingFetch; + expect(response.status).toBe(502); + await expect(response.json()).resolves.toEqual({ error: testCase.error }); + } + }, 20000); }); diff --git a/packages/adapter-utils/src/duplex-frame-codec.test.ts b/packages/adapter-utils/src/duplex-frame-codec.test.ts index c19008fb558..0fddd5e3cc2 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.test.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.test.ts @@ -126,7 +126,7 @@ describe("round-trip", () => { ), ), ); - for (const type of ["request", "response", "ready", "heartbeat", "close", "error"]) { + for (const type of ["request", "response", "body_chunk", "ready", "heartbeat", "close", "error"]) { expect(types).toContain(type); } }); @@ -212,8 +212,8 @@ describe("streaming decoder behavior", () => { method: "POST", path: "/x", query: "", - headers: {}, - body: "😀", + headers: { "x-note": "😀" }, + bodyByteCount: 0, }; const bytes = Buffer.from(encodeDuplexFrame(frame), "utf8"); // Split inside the four-byte emoji, right before a continuation byte. @@ -275,7 +275,7 @@ describe("request id byte bound", () => { path: "/", query: "", headers: {}, - body: "", + bodyByteCount: 0, }); } @@ -314,7 +314,7 @@ describe("request id byte bound", () => { id, status: 200, headers: {}, - body: "", + bodyByteCount: 0, outcome: "completed", }), ); @@ -461,8 +461,8 @@ describe("size-checked encode", () => { type: "response", id: "big", status: 200, - headers: {}, - body: "x".repeat(2_000), + headers: { "x-pad": "x".repeat(2_000) }, + bodyByteCount: 0, outcome: "completed", }; expect(() => encodeDuplexFrameChecked(frame, 1_000)).not.toThrow(); @@ -475,22 +475,25 @@ describe("size-checked encode", () => { // Build a frame whose encoded JSON is exactly N bytes. The guard measures the // JSON without the newline, so N bytes encodes and N+1 bytes fails. This is the // same boundary the decoder applies, so encode and decode agree exactly. - const base: DuplexFrame = { + // Pad a header value to grow the encoded JSON by an exact byte count. The + // "x-pad" value adds one byte per character, so the frame lands on an exact + // size the guard measures without the trailing newline. + const pad = (n: number): DuplexFrame => ({ version: DUPLEX_FRAME_VERSION, type: "response", id: "boundary", status: 200, - headers: {}, - body: "", + headers: { "x-pad": "x".repeat(n) }, + bodyByteCount: 0, outcome: "completed", - }; - const baseBytes = Buffer.byteLength(JSON.stringify(base), "utf8"); + }); + const baseBytes = Buffer.byteLength(JSON.stringify(pad(0)), "utf8"); const limit = baseBytes + 10; - const atLimit: DuplexFrame = { ...base, body: "x".repeat(10) }; + const atLimit = pad(10); expect(Buffer.byteLength(JSON.stringify(atLimit), "utf8")).toBe(limit); const okResult = encodeDuplexFrameChecked(atLimit, limit); expect(okResult.ok).toBe(true); - const overResult = encodeDuplexFrameChecked({ ...base, body: "x".repeat(11) }, limit); + const overResult = encodeDuplexFrameChecked(pad(11), limit); expect(overResult.ok).toBe(false); if (!overResult.ok) expect(overResult.error.code).toBe("frame_too_large"); }); diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index c1723ad763c..1013dbf106c 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -24,7 +24,15 @@ import { } from "./duplex-aggregate-byte-ledger.js"; /** The wire version this codec reads and writes. */ -export const DUPLEX_FRAME_VERSION = 1; +export const DUPLEX_FRAME_VERSION = 2; + +/** + * The fixed raw byte size of one body slice. The sender splits a body into + * `body_chunk` frames of this raw size, except the final chunk, which holds the + * remaining bytes. The base64 text of one full slice is about 349 KiB, so each + * encoded `body_chunk` frame stays well under {@link DEFAULT_MAX_DUPLEX_FRAME_BYTES}. + */ +export const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024; /** * The default maximum size of one frame, in bytes. The decoder rejects a longer @@ -54,6 +62,7 @@ const EMPTY = Buffer.alloc(0); export const DUPLEX_FRAME_TYPES = { request: "request", response: "response", + body_chunk: "body_chunk", ready: "ready", heartbeat: "heartbeat", close: "close", @@ -63,7 +72,12 @@ export const DUPLEX_FRAME_TYPES = { /** The outcome of a response frame. A loss response carries a non-completed outcome. */ export type DuplexResponseOutcome = "completed" | "indeterminate" | "unavailable"; -/** A request frame. The gateway forwards it to the host API path. */ +/** + * A request envelope frame. The gateway forwards it to the host API path. The + * envelope carries `bodyByteCount`, the raw byte length of the request body. The + * body itself rides one or more `body_chunk` frames that share the same `id`. A + * `bodyByteCount` of 0 means an empty body and no `body_chunk` frame follows. + */ export interface DuplexRequestFrame { version: number; type: "request"; @@ -72,20 +86,43 @@ export interface DuplexRequestFrame { path: string; query: string; headers: Record; - body: string; + bodyByteCount: number; } -/** A response frame. The host returns it for one request id. */ +/** + * A response envelope frame. The host returns it for one request id. The + * envelope carries `bodyByteCount`, the raw byte length of the response body. + * The body itself rides one or more `body_chunk` frames that share the same + * `id`. A `bodyByteCount` of 0 means an empty body and no `body_chunk` frame + * follows. + */ export interface DuplexResponseFrame { version: number; type: "response"; id: string; status: number; headers: Record; - body: string; + bodyByteCount: number; outcome: DuplexResponseOutcome; } +/** + * A body-chunk frame. One frame carries one raw body slice as base64 text in + * `data`. The frames for one body share the `id` of the envelope. `seq` starts + * at 0 and increases by one for each next chunk, with no gap and no repeat. Each + * raw slice is {@link DUPLEX_BODY_CHUNK_RAW_BYTES} bytes, except the final slice, + * which holds the remaining bytes. The receiver reassembles the slices in `seq` + * order and stops when the total raw byte count equals the envelope + * `bodyByteCount`. + */ +export interface DuplexBodyChunkFrame { + version: number; + type: "body_chunk"; + id: string; + seq: number; + data: string; +} + /** * The READY control frame. The gateway sends it one time after it binds the * host-assigned listener port. READY is a liveness signal, not an address @@ -129,6 +166,7 @@ export interface DuplexErrorFrame { export type DuplexFrame = | DuplexRequestFrame | DuplexResponseFrame + | DuplexBodyChunkFrame | DuplexReadyFrame | DuplexHeartbeatFrame | DuplexCloseFrame @@ -194,6 +232,11 @@ function idWithinLimit(id: string): boolean { return Buffer.byteLength(id, "utf8") <= DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES; } +/** Return true when the value is a safe integer that is zero or positive. */ +function isSafeNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + /** * Encode one frame to a single line of JSON with a trailing newline. `JSON.stringify` * escapes any newline inside a string value, so the returned line holds no @@ -261,6 +304,8 @@ function validateFrame(frame: Record): DuplexDecodeResult { return validateRequest(frame); case "response": return validateResponse(frame); + case "body_chunk": + return validateBodyChunk(frame); case "ready": return validateReady(frame); case "heartbeat": @@ -280,7 +325,7 @@ function validateRequest(frame: Record): DuplexDecodeResult { typeof frame.method !== "string" || typeof frame.path !== "string" || typeof frame.query !== "string" || - typeof frame.body !== "string" || + !isSafeNonNegativeInteger(frame.bodyByteCount) || !isStringRecord(frame.headers) ) { return fail("malformed_frame", "request frame has a missing or wrong-typed field"); @@ -298,7 +343,7 @@ function validateResponse(frame: Record): DuplexDecodeResult { if ( typeof frame.id !== "string" || typeof frame.status !== "number" || - typeof frame.body !== "string" || + !isSafeNonNegativeInteger(frame.bodyByteCount) || !isStringRecord(frame.headers) || typeof frame.outcome !== "string" || !RESPONSE_OUTCOMES.has(frame.outcome) @@ -313,6 +358,29 @@ function validateResponse(frame: Record): DuplexDecodeResult { return ok(frame as unknown as DuplexResponseFrame); } +/** + * Validate a `body_chunk` frame at the structural level. The codec is stateless, + * so it checks only the per-frame shape here: + * - `id` is a string within {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}. + * - `seq` is a safe integer that is zero or positive. + * - `data` is a string. + * The receive-side reassembler owns the stateful checks: the base64 canonical + * form, the fixed slice size, the `seq` order, and the total against + * `bodyByteCount`. The codec never decodes `data` here. + */ +function validateBodyChunk(frame: Record): DuplexDecodeResult { + if (typeof frame.id !== "string" || typeof frame.data !== "string") { + return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed field"); + } + if (!isSafeNonNegativeInteger(frame.seq)) { + return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq"); + } + if (!idWithinLimit(frame.id)) { + return fail("id_too_large", "body_chunk frame id exceeds the maximum size"); + } + return ok(frame as unknown as DuplexBodyChunkFrame); +} + function validateReady(frame: Record): DuplexDecodeResult { // READY carries a liveness nonce, not an address. The schema is strict: a valid // READY frame holds exactly `version`, `type`, and `nonce`. The decoder rejects diff --git a/packages/adapter-utils/src/duplex-frame-vectors.json b/packages/adapter-utils/src/duplex-frame-vectors.json index 5bedc93081e..62d43d1c15f 100644 --- a/packages/adapter-utils/src/duplex-frame-vectors.json +++ b/packages/adapter-utils/src/duplex-frame-vectors.json @@ -1,17 +1,17 @@ { - "frameVersion": 1, + "frameVersion": 2, "defaultMaxFrameBytes": 1000000, "description": "Shared wire-compatibility vectors for the duplex frame codec. Every codec copy decodes the same bytes. bytes is a UTF-8 byte stream; splitByteOffsets cuts it into chunks; expected lists the ordered decode results (frame or protocol-error code).", "vectors": [ { "name": "valid-request", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -21,7 +21,30 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 + } + } + ] + }, + { + "name": "valid-request-with-body-count", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-body\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":1048576}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "request", + "id": "req-body", + "method": "GET", + "path": "/api/issues/PAP-1", + "query": "expand=comments", + "headers": { + "accept": "application/json", + "content-type": "application/json" + }, + "bodyByteCount": 1048576 } } ] @@ -29,19 +52,19 @@ { "name": "valid-response-completed", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-1", "status": 200, "headers": { "content-type": "application/json" }, - "body": "{\"ok\":true}", + "bodyByteCount": 11, "outcome": "completed" } } @@ -50,19 +73,19 @@ { "name": "valid-response-indeterminate", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"outcome_indeterminate\\\"}\",\"outcome\":\"indeterminate\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":33,\"outcome\":\"indeterminate\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-2", "status": 409, "headers": { "content-type": "application/json" }, - "body": "{\"error\":\"outcome_indeterminate\"}", + "bodyByteCount": 33, "outcome": "indeterminate" } } @@ -71,43 +94,121 @@ { "name": "valid-response-unavailable", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"bridge_unavailable\\\"}\",\"outcome\":\"unavailable\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":30,\"outcome\":\"unavailable\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-3", "status": 503, "headers": { "content-type": "application/json" }, - "body": "{\"error\":\"bridge_unavailable\"}", + "bodyByteCount": 30, "outcome": "unavailable" } } ] }, + { + "name": "valid-body-chunk", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":\"aGVsbG8gYm9keSBjaHVuaw==\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-1", + "seq": 0, + "data": "aGVsbG8gYm9keSBjaHVuaw==" + } + } + ] + }, + { + "name": "valid-body-chunk-final", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":7,\"data\":\"dGFpbA==\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-1", + "seq": 7, + "data": "dGFpbA==" + } + } + ] + }, { "name": "valid-ready", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "ready", "nonce": "9f8e7d6c5b4a39281706f5e4d3c2b1a0" } } ] }, + { + "name": "valid-heartbeat", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"heartbeat\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "heartbeat" + } + } + ] + }, + { + "name": "valid-close", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"close\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "close" + } + } + ] + }, + { + "name": "valid-error", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "error", + "code": "malformed_frame", + "message": "peer reported a malformed frame" + } + } + ] + }, { "name": "invalid-ready-missing-nonce", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\"}\n", "expected": [ { "error": "malformed_frame" @@ -117,7 +218,7 @@ { "name": "invalid-ready-with-address", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n", "expected": [ { "error": "malformed_frame" @@ -127,57 +228,13 @@ { "name": "invalid-ready-with-port", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n", "expected": [ { "error": "malformed_frame" } ] }, - { - "name": "valid-heartbeat", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"heartbeat\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "heartbeat" - } - } - ] - }, - { - "name": "valid-close", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"close\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "close" - } - } - ] - }, - { - "name": "valid-error", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "error", - "code": "malformed_frame", - "message": "peer reported a malformed frame" - } - } - ] - }, { "name": "invalid-not-json", "category": "invalid", @@ -211,7 +268,7 @@ { "name": "invalid-unknown-type", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"teleport\"}\n", + "bytes": "{\"version\":2,\"type\":\"teleport\"}\n", "expected": [ { "error": "unknown_type" @@ -221,7 +278,37 @@ { "name": "invalid-missing-id", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-string", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":\"10\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-negative", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":-1}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-fractional", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":1.5}\n", "expected": [ { "error": "malformed_frame" @@ -231,7 +318,47 @@ { "name": "invalid-bad-headers", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"bodyByteCount\":10,\"outcome\":\"completed\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-negative-seq", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":-1,\"data\":\"eA==\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-fractional-seq", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":2.5,\"data\":\"eA==\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-missing-data", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-nonstring-data", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":42}\n", "expected": [ { "error": "malformed_frame" @@ -241,14 +368,14 @@ { "name": "partial-request-split", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "splitByteOffsets": [ - 98 + 102 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -258,7 +385,7 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 } } ] @@ -266,23 +393,24 @@ { "name": "partial-utf8-split", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1/comments\",\"query\":\"\",\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"text\\\":\\\"héllo 😀 wörld\\\"}\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"content-type\":\"application/json\",\"x-note\":\"héllo 😀 wörld\"},\"bodyByteCount\":0}\n", "splitByteOffsets": [ - 177 + 172 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-utf8", "method": "POST", - "path": "/api/issues/PAP-1/comments", - "query": "", + "path": "/api/issues/PAP-1", + "query": "expand=comments", "headers": { - "content-type": "application/json" + "content-type": "application/json", + "x-note": "héllo 😀 wörld" }, - "body": "{\"text\":\"héllo 😀 wörld\"}" + "bodyByteCount": 0 } } ] @@ -290,16 +418,16 @@ { "name": "partial-two-frames", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n", "splitByteOffsets": [ 7, - 199, - 343 + 174, + 344 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -309,19 +437,19 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 } }, { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-1", "status": 200, "headers": { "content-type": "application/json" }, - "body": "{\"ok\":true}", + "bodyByteCount": 11, "outcome": "completed" } } @@ -330,14 +458,14 @@ { "name": "partial-no-trailing-newline", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"heartbeat\"}", + "bytes": "{\"version\":2,\"type\":\"heartbeat\"}", "expected": [] }, { "name": "oversized-complete-line", "category": "oversized", "maxFrameBytes": 64, - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"q=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "expected": [ { "error": "frame_too_large" @@ -351,23 +479,33 @@ "splitByteOffsets": [ 40 ], - "bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":1,\"type\":\"heartbeat\"}\n", + "bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":2,\"type\":\"heartbeat\"}\n", "expected": [ { "error": "frame_too_large" }, { "frame": { - "version": 1, + "version": 2, "type": "heartbeat" } } ] }, + { + "name": "version-mismatch-lower", + "category": "versionMismatch", + "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "version_mismatch" + } + ] + }, { "name": "version-mismatch-higher", "category": "versionMismatch", - "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":3,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "expected": [ { "error": "version_mismatch" @@ -387,19 +525,19 @@ { "name": "request-id-at-max-bytes", "category": "idBoundValid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "method": "GET", "path": "/", "query": "", "headers": {}, - "body": "" + "bodyByteCount": 0 } } ] @@ -407,7 +545,17 @@ { "name": "request-id-over-max-bytes", "category": "idBoundInvalid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "id_too_large" + } + ] + }, + { + "name": "body-chunk-id-over-max-bytes", + "category": "idBoundInvalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"seq\":0,\"data\":\"eA==\"}\n", "expected": [ { "error": "id_too_large" @@ -421,14 +569,28 @@ "name": "encode-within-bound", "maxFrameBytes": 200, "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-enc-1", "method": "GET", "path": "/", "query": "", "headers": {}, - "body": "small" + "bodyByteCount": 5 + }, + "expected": { + "ok": true + } + }, + { + "name": "encode-body-chunk-within-bound", + "maxFrameBytes": 200, + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-enc-3", + "seq": 0, + "data": "Y2h1bmsgcGF5bG9hZA==" }, "expected": { "ok": true @@ -438,7 +600,7 @@ "name": "encode-control-frame-never-rejected", "maxFrameBytes": 200, "frame": { - "version": 1, + "version": 2, "type": "heartbeat" }, "expected": { @@ -449,14 +611,15 @@ "name": "encode-over-bound", "maxFrameBytes": 200, "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-enc-2", "status": 200, "headers": { - "content-type": "application/json" + "content-type": "application/json", + "x-pad": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, - "body": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "bodyByteCount": 0, "outcome": "completed" }, "expected": { diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 0ecf167102c..b5c5c6b1481 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -61,6 +61,7 @@ import { type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import { assertNestedDuplexBrokerBudgets, createDuplexBridgeBroker, @@ -92,6 +93,51 @@ import { const execFileAsync = promisify(execFile); +/** + * Emit one duplex request the way the sandbox gateway sends it under the chunked + * body protocol: the envelope frame that carries `bodyByteCount`, then the + * `body_chunk` frames that carry the body. A non-empty body rides one or more + * `body_chunk` frames that share the envelope id; an empty body sends no chunk. + */ +function emitDuplexRequest( + control: { emitData: (chunk: string) => void }, + frame: Omit, + bodyText: string, +): void { + const body = Buffer.from(bodyText, "utf8"); + control.emitData(encodeDuplexFrame({ ...frame, bodyByteCount: body.length })); + for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { + control.emitData(encodeDuplexFrame(chunk)); + } +} + +/** + * Send one duplex response the way the host broker sends it under the chunked + * body protocol: the envelope frame that carries `bodyByteCount`, then the + * `body_chunk` frames that carry the body. The gateway reassembles the body from + * the chunk frames before it writes the HTTP response. + */ +function sendDuplexResponse( + send: (frame: Record) => void, + response: { id: string | undefined; status: number; headers: Record; outcome?: string }, + bodyText: string, +): void { + const id = response.id ?? ""; + const body = Buffer.from(bodyText, "utf8"); + send({ + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: response.status, + headers: response.headers, + bodyByteCount: body.length, + ...(response.outcome !== undefined ? { outcome: response.outcome } : {}), + }); + for (const chunk of splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION)) { + send(chunk as unknown as Record); + } +} + type RecordedSpan = { name: string; parentName: string | null; ended: boolean }; /** @@ -3140,7 +3186,7 @@ describe("sandbox adapter execution targets", () => { if (onOpen) { onOpen({ nonce, port, emitRaw, emitFrame, emitExit }); } else { - emitFrame({ version: 1, type: "ready", nonce }); + emitFrame({ version: 2, type: "ready", nonce }); } }); }, @@ -3250,8 +3296,9 @@ describe("sandbox adapter execution targets", () => { // The gateway forwards one agent request as a request frame. The broker // forwards it on the host path with the real token and the run id, then // writes one response frame back. - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-1", @@ -3259,8 +3306,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -3304,7 +3351,7 @@ describe("sandbox adapter execution targets", () => { // The flood is larger than the ceiling; the READY frame is far smaller. const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4096 }); const { runner, control } = makeDuplexSelectionRunner((ctx) => { - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); ctx.emitRaw("x".repeat(64 * 1024)); }); const { recorder, counters } = createRecordingDuplexRecorder(); @@ -3557,15 +3604,15 @@ describe("sandbox adapter execution targets", () => { { name: "a mismatched nonce", onOpen: (ctx: DuplexOpenContext) => - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }), + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }), }, { name: "an incomplete READY frame", - onOpen: (ctx: DuplexOpenContext) => ctx.emitRaw('{"version":1,"type":"ready"}\n'), + onOpen: (ctx: DuplexOpenContext) => ctx.emitRaw('{"version":2,"type":"ready"}\n'), }, { name: "protocol contamination before READY", - onOpen: (ctx: DuplexOpenContext) => ctx.emitFrame({ version: 1, type: "heartbeat" }), + onOpen: (ctx: DuplexOpenContext) => ctx.emitFrame({ version: 2, type: "heartbeat" }), }, { name: "a gateway bind failure with no READY frame", @@ -3619,12 +3666,12 @@ describe("sandbox adapter execution targets", () => { { name: "an attacker-owned numeric local port", buildReady: (nonce: string, attackerPort: number) => - `{"version":1,"type":"ready","nonce":"${nonce}","port":${attackerPort}}\n`, + `{"version":2,"type":"ready","nonce":"${nonce}","port":${attackerPort}}\n`, }, { name: "a channel-supplied host URL", buildReady: (nonce: string, attackerPort: number) => - `{"version":1,"type":"ready","nonce":"${nonce}","address":"http://127.0.0.1:${attackerPort}"}\n`, + `{"version":2,"type":"ready","nonce":"${nonce}","address":"http://127.0.0.1:${attackerPort}"}\n`, }, ])( "rejects a READY frame that carries $name and never sends the bridge token there", @@ -3724,8 +3771,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-forbidden", @@ -3733,8 +3781,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/secret-admin-route", query: "", headers: { authorization: "Bearer bridge-token", "content-type": "application/json" }, - body: JSON.stringify({ escalate: true }), - }), + }, + JSON.stringify({ escalate: true }), ); await waitForCondition( () => control.written.length >= 1, @@ -3850,8 +3898,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-obs", @@ -3859,8 +3908,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -4060,8 +4109,9 @@ describe("sandbox adapter execution targets", () => { try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); if (dispatchFirst) { - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-loss", @@ -4069,8 +4119,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -4126,8 +4176,9 @@ describe("sandbox adapter execution targets", () => { try { // The throwing recorder never blocked the duplex selection. expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-guard", @@ -4135,8 +4186,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -4192,7 +4243,7 @@ describe("sandbox adapter execution targets", () => { onData(listener: (chunk: string) => void): void { dataListener = listener; control.emitData = (chunk) => dataListener?.(chunk); - setImmediate(() => dataListener?.(`${JSON.stringify({ version: 1, type: "ready", nonce })}\n`)); + setImmediate(() => dataListener?.(`${JSON.stringify({ version: 2, type: "ready", nonce })}\n`)); }, onExit(_listener: (exit: { exitCode: number | null }) => void): void {}, stop(): void {}, @@ -4237,8 +4288,9 @@ describe("sandbox adapter execution targets", () => { // A request that carries the sentinel route, query, body, and bridge token. // The response write then fails with the sentinel provider error. - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-redact", @@ -4246,8 +4298,8 @@ describe("sandbox adapter execution targets", () => { path: `/api/${ROUTE_SENTINEL}`, query: `secret=${QUERY_SENTINEL}`, headers: { authorization: `Bearer ${BRIDGE_TOKEN_SENTINEL}` }, - body: BODY_SENTINEL, - }), + }, + BODY_SENTINEL, ); // Give the forward and the failing response write time to run and record a loss. await waitForCondition( @@ -4303,8 +4355,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-prov", @@ -4312,8 +4365,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => spans.some((s) => s.name === DUPLEX_SPAN_REQUEST), @@ -4527,7 +4580,7 @@ describe("sandbox adapter execution targets", () => { const reserveSpy = vi.spyOn(ledger, "reserve"); const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("pty-echo-noise"); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); const target: AdapterSandboxExecutionTarget = { kind: "remote", @@ -4644,9 +4697,9 @@ describe("sandbox adapter execution targets", () => { const noisePrefix = "\n".repeat(blankLineCount) + "a non-frame echo line\n"; const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw(noisePrefix); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); - const readyLine = '{"version":1,"type":"ready","nonce":""}\n'; + const readyLine = '{"version":2,"type":"ready","nonce":""}\n'; const totalBytes = noisePrefix.length + readyLine.length; const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4701,8 +4754,8 @@ describe("sandbox adapter execution targets", () => { // must skip the echo line and a partial-JSON line, then accept the READY frame. const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("sh -c exec env PAPERCLIP_BRIDGE_NONCE=... node gateway.mjs\n"); - ctx.emitRaw('{"version":1,"type":"ready"}\n'); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitRaw('{"version":2,"type":"ready"}\n'); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4752,7 +4805,7 @@ describe("sandbox adapter execution targets", () => { // the `ready_nonce_mismatch` reason. const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("a non-frame echo line\n"); - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }); + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }); }); const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4808,7 +4861,7 @@ describe("sandbox adapter execution targets", () => { // per-skip cap check stops the bypass. const readinessBufferCapBytes = DEFAULT_MAX_DUPLEX_FRAME_BYTES + 4_096; const { runner, control } = makeDuplexSelectionRunner((ctx) => { - const readyLine = `${JSON.stringify({ version: 1, type: "ready", nonce: ctx.nonce })}\n`; + const readyLine = `${JSON.stringify({ version: 2, type: "ready", nonce: ctx.nonce })}\n`; ctx.emitRaw("\n".repeat(readinessBufferCapBytes + 1) + readyLine); }); const { recorder, counters } = createRecordingDuplexRecorder(); @@ -4860,7 +4913,7 @@ describe("sandbox adapter execution targets", () => { // carries the closed `fallback_reason` dimension, so a reader can group the // failed opens by reason. const { runner } = makeDuplexSelectionRunner((ctx) => - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }), + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }), ); const { recorder, spans } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4930,8 +4983,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-hdr", @@ -4946,8 +5000,8 @@ describe("sandbox adapter execution targets", () => { // A sandbox-supplied auth header the host must replace with the real token. authorization: "Bearer bridge-token", }, - body: "", - }), + }, + "", ); await waitForCondition( () => api.requests.length >= 1, @@ -5000,8 +5054,9 @@ describe("sandbox adapter execution targets", () => { try { // The broker started with derived nested budgets, so the duplex transport serves. expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-budget", @@ -5009,8 +5064,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -5120,7 +5175,7 @@ describe("sandbox adapter execution targets", () => { "daytona@212487a7f3c9:~$ exec 2>'/tmp/paperclip-duplex-x.log'; stty raw -echo; " + "exec 'bash' '-c' 'exec env PAPERCLIP_BRIDGE_NONCE=" + ctx.nonce + " node gateway.mjs'\r\n", ); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -5137,7 +5192,7 @@ describe("sandbox adapter execution targets", () => { "stty raw -echo; exec 'bash' '-c' 'exec env node gateway.mjs'\r\n", ); // No newline between the escape sequence and the frame: same line. - ctx.emitRaw('\x1b[?2004l\r{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('\x1b[?2004l\r{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -5146,7 +5201,7 @@ describe("sandbox adapter execution targets", () => { it("PTY replay: accepts READY split across chunk boundaries", async () => { const { mode } = await runReadinessReplay((ctx) => { ctx.emitRaw("prompt$ wrapper-line\r\n"); - const frame = '{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'; + const frame = '{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'; ctx.emitRaw(frame.slice(0, 12)); ctx.emitRaw(frame.slice(12)); }); @@ -5159,7 +5214,7 @@ describe("sandbox adapter execution targets", () => { const noise = Buffer.from("prompt ✓ done\r\n", "utf8"); ctx.emitRaw(noise.slice(0, 8).toString("utf8")); ctx.emitRaw(noise.slice(8).toString("utf8")); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -5170,7 +5225,7 @@ describe("sandbox adapter execution targets", () => { const { mode } = await runReadinessReplay((ctx) => { const line = "x".repeat(64) + "\n"; for (let i = 0; i < 80_000; i += 1) ctx.emitRaw(line); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("queue_v1"); }, 30000); @@ -5280,7 +5335,7 @@ describe("sandbox duplex gateway", () => { // process the launch replaced the shell with. await writeFile( stub, - '#!/bin/sh\nprintf \'{"version":1,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', + '#!/bin/sh\nprintf \'{"version":2,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', "utf8", ); const argv = buildDuplexGatewayLaunchArgv({ @@ -5641,15 +5696,16 @@ describe("sandbox duplex gateway", () => { "if-none-match": '"cache-key"', }); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, - body: JSON.stringify({ ok: true }), - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, + outcome: "completed", + }, + JSON.stringify({ ok: true }), + ); const response = await responsePromise; expect(response.status).toBe(200); expect(response.headers.get("content-type")).toContain("application/json"); @@ -5666,42 +5722,45 @@ describe("sandbox duplex gateway", () => { const patchFrame = await gateway.waitForFrame( (frame) => frame.type === "request" && frame.id !== requestFrame.id, ); - gateway.sendFrame({ - version: 1, - type: "response", - id: patchFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: "outcome_indeterminate" }), - outcome: "indeterminate", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: patchFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "indeterminate", + }, + JSON.stringify({ error: "outcome_indeterminate" }), + ); const indeterminate = await indeterminatePromise; expect(indeterminate.status).toBe(409); await gateway.stop(); }, 20000); - it("fails a request that exceeds the frame size bound with a local 413 and keeps the channel open", async () => { + it("fails a request over the body size limit locally and keeps the channel open", async () => { const token = "duplex-token-oversize-request"; - // Raise the body limit above the frame bound, so `readBody` accepts the body - // and the encode guard is the only limit the request meets. The default frame - // bound is 1,000,000 bytes. + // The chunked body protocol splits a large body into fixed-size body_chunk + // frames that each stay under the frame bound, so a body never exceeds the + // frame bound on its own. The operative local guard for a request too large to + // deliver is now the body size limit. Set it to 1,000,000 bytes here. const gateway = await startDuplexGateway({ PAPERCLIP_BRIDGE_TOKEN: token, - PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "3000000", + PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "1000000", }); - // A body over the frame bound makes the request frame exceed the bound. The - // gateway must fail this one local request with a clean 413 and forward no - // frame. + // A body over the body size limit is rejected locally. The gateway must fail + // this one local request cleanly and forward no frame. const oversizeBody = JSON.stringify({ body: "x".repeat(1_100_000) }); const tooLarge = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: oversizeBody, }); - expect(tooLarge.status).toBe(413); - await expect(tooLarge.json()).resolves.toEqual({ error: "request_too_large" }); + expect(tooLarge.status).toBe(502); + await expect(tooLarge.json()).resolves.toEqual({ + error: "Bridge request body exceeded the configured size limit.", + }); // The oversized request forwarded no frame. It never left the gateway. expect(gateway.frames.filter((frame) => frame.type === "request")).toHaveLength(0); @@ -5714,15 +5773,16 @@ describe("sandbox duplex gateway", () => { headers: { authorization: `Bearer ${token}` }, }); const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ ok: true }), - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "completed", + }, + JSON.stringify({ ok: true }), + ); const okResponse = await okPromise; expect(okResponse.status).toBe(200); await expect(okResponse.json()).resolves.toEqual({ ok: true }); @@ -5859,24 +5919,21 @@ describe("sandbox duplex gateway", () => { // Feed a malformed inbound line and an unknown response id. Both force a // diagnostic path; none of it may reach stdout. gateway.sendRaw("this is not a frame\n"); - gateway.sendFrame({ - version: 1, - type: "response", - id: "unknown-id", - status: 200, - headers: {}, - body: "", - outcome: "completed", - }); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: "{}", - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { id: "unknown-id", status: 200, headers: {}, outcome: "completed" }, + "", + ); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "completed", + }, + "{}", + ); const response = await responsePromise; expect(response.status).toBe(200); @@ -5932,7 +5989,7 @@ function createFakeDuplexChannel(): { channel: CommandManagedDuplexChannel; emitData: (chunk: string) => void; emitExit: (exit: { exitCode: number | null }) => void; - written: DuplexResponseFrame[]; + written: Array; writtenTypes: string[]; stopped: () => number; setWriteError: (error: Error | null) => void; @@ -5943,16 +6000,43 @@ function createFakeDuplexChannel(): { let writeError: Error | null = null; let closeBehavior: "resolve" | "hang" | "reject" = "resolve"; let stopCount = 0; - const written: DuplexResponseFrame[] = []; + // The broker writes a response as one envelope frame that carries + // `bodyByteCount`, then the `body_chunk` frames that carry the body. Reassemble + // the body so a test reads the response the same way it did with the one-frame + // model. + const written: Array = []; const writtenTypes: string[] = []; + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); const channel: CommandManagedDuplexChannel = { write(data: string): void { if (writeError) throw writeError; const decoded = decodeDuplexLine(data.replace(/\n$/, "")); - if (decoded.ok) { - writtenTypes.push(decoded.frame.type); - if (decoded.frame.type === "response") written.push(decoded.frame); + if (!decoded.ok) return; + const frame = decoded.frame; + writtenTypes.push(frame.type); + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + written.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) return; + const decodedChunk = Buffer.from(frame.data, "base64"); + assembly.received += decodedChunk.length; + assembly.chunks.push(decodedChunk); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + written.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } } }, onData(listener: (chunk: string) => void): void { @@ -5987,8 +6071,16 @@ function createFakeDuplexChannel(): { }; } -/** Build one request frame line the fake channel can emit. */ -function requestFrameLine(overrides: Partial & { id: string }): string { +/** + * Build one request as its envelope line (carrying `bodyByteCount`) plus its + * `body_chunk` lines, so the fake channel emits a whole request in one write. The + * broker reassembles the body from the chunk frames before it forwards. + */ +function requestFrameLine( + overrides: Partial> & { id: string }, + bodyText: string = JSON.stringify({ body: "hello" }), +): string { + const body = Buffer.from(bodyText, "utf8"); const frame: DuplexRequestFrame = { version: DUPLEX_FRAME_VERSION, type: "request", @@ -5997,9 +6089,13 @@ function requestFrameLine(overrides: Partial & { id: string path: overrides.path ?? "/api/issues/PAP-1/comments", query: overrides.query ?? "", headers: overrides.headers ?? { authorization: "Bearer bridge-token" }, - body: overrides.body ?? JSON.stringify({ body: "hello" }), + bodyByteCount: body.length, }; - return encodeDuplexFrame(frame); + let line = encodeDuplexFrame(frame); + for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { + line += encodeDuplexFrame(chunk); + } + return line; } /** Wait for the pending microtasks and macrotasks to settle. */ @@ -6015,7 +6111,7 @@ describe("createDuplexBridgeBroker", () => { // broker passes the decoded request straight through, so the sandbox request // still carries only the bridge token here, and the handler applies the real // token and the signed run identifier on the existing forward path. - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request): Promise => { received.push(request); @@ -6050,7 +6146,7 @@ describe("createDuplexBridgeBroker", () => { it("moves through opening, open, closing, closed in order", async () => { const fake = createFakeDuplexChannel(); const states: DuplexBrokerState[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), onStateChange: (state) => states.push(state), @@ -6089,7 +6185,7 @@ describe("createDuplexBridgeBroker", () => { ])("enters lost on $name", async ({ reason, trigger }) => { const fake = createFakeDuplexChannel(); const losses: DuplexBrokerLossRecord[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), onLoss: (record) => losses.push(record), @@ -6107,7 +6203,7 @@ describe("createDuplexBridgeBroker", () => { it("enters lost on a close timeout", async () => { const fake = createFakeDuplexChannel(); fake.setCloseBehavior("hang"); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), closeTimeoutMs: 20, @@ -6123,7 +6219,7 @@ describe("createDuplexBridgeBroker", () => { it("stops the heartbeat, marks the run bridge ended, and dispatches nothing after loss", async () => { const fake = createFakeDuplexChannel(); const forwarded: string[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request) => { forwarded.push(request.id); @@ -6147,7 +6243,7 @@ describe("createDuplexBridgeBroker", () => { it("forwards one request id one time, so a repeated frame never reaches the API twice", async () => { const fake = createFakeDuplexChannel(); const forwarded: string[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request) => { forwarded.push(request.id); @@ -6171,7 +6267,7 @@ describe("createDuplexBridgeBroker", () => { const fake = createFakeDuplexChannel(); const records: DuplexBrokerRequestRecord[] = []; let clock = 1000; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), now: () => clock, @@ -6194,7 +6290,7 @@ describe("createDuplexBridgeBroker", () => { it("latches a failure when a loss orders before an orderly completion and names the typed reason", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6227,7 +6323,7 @@ describe("createDuplexBridgeBroker", () => { }, }; const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), telemetry: createDuplexTelemetry({ recorder, providerKey: "daytona" }), @@ -6263,7 +6359,7 @@ describe("createDuplexBridgeBroker", () => { const logLines: string[] = []; const fake = createFakeDuplexChannel(); const sentinel = "SENTINEL-PROVIDER-TEXT-1a2b3c"; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), telemetry: createDuplexTelemetry({ recorder, providerKey: "daytona" }), @@ -6287,7 +6383,7 @@ describe("createDuplexBridgeBroker", () => { expect(logLines.join("\n")).not.toContain(sentinel); }); - it("rejects a configuration where an inner budget is not smaller than its outer budget", () => { + it("rejects a configuration where an inner budget is not smaller than its outer budget", async () => { expect(() => assertNestedDuplexBrokerBudgets({ forwardTimeoutMs: 32_000, @@ -6302,13 +6398,15 @@ describe("createDuplexBridgeBroker", () => { gatewayWaitMs: 35_000, }), ).toThrow(/response budget/); - expect(() => + // The async broker construction validates budgets before its first await, so + // an invalid budget set surfaces as a rejected promise. + await expect( createDuplexBridgeBroker({ channel: createFakeDuplexChannel().channel, forwardRequest: async () => ({ status: 200 }), budgets: { forwardTimeoutMs: 40_000 }, }), - ).toThrow(/forward budget/); + ).rejects.toThrow(/forward budget/); // The default budget set holds the nested order. expect(() => assertNestedDuplexBrokerBudgets({ @@ -6384,7 +6482,7 @@ describe("sandbox target spec parse: enableSandboxDuplexBridge", () => { describe("settleRunDisposition atomic read and mark", () => { it("marks the orderly completion and reports a success for a healthy channel", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6400,7 +6498,7 @@ describe("settleRunDisposition atomic read and mark", () => { it("reports the failure and does not mark for a latched loss", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6443,8 +6541,8 @@ describe("CLI-lane run-disposition seam", () => { } as AdapterSandboxExecutionTarget; } - function startBroker(fake: ReturnType) { - const broker = createDuplexBridgeBroker({ + async function startBroker(fake: ReturnType) { + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -6454,7 +6552,7 @@ describe("CLI-lane run-disposition seam", () => { it("fails a clean CLI completion closed when the duplex channel was lost mid-turn", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The control channel dies mid-turn, before the CLI process exits. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6478,7 +6576,7 @@ describe("CLI-lane run-disposition seam", () => { it("keeps a clean CLI completion a success when the channel stays healthy, and a teardown loss stays benign", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); const runner = mockRunner({ ...CLEAN_RESULT }); const result = await runAdapterExecutionTargetProcess("run-cli-ok", sandboxTarget(runner), "agent-cli", [], { @@ -6502,7 +6600,7 @@ describe("CLI-lane run-disposition seam", () => { it("keeps a clean CLI completion a success when the gateway exits during the run-log tail finish", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // A run-log tail whose finish emits a gateway exit. This reproduces the // race where the duplex gateway dies after the clean process completion but @@ -6542,7 +6640,7 @@ describe("CLI-lane run-disposition seam", () => { it("cannot clear the loss latch with a later completion", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The loss latches before the CLI process exits. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6565,7 +6663,7 @@ describe("CLI-lane run-disposition seam", () => { it("leaves an already-failed CLI result unchanged and never settles the disposition", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The control channel is lost, but the process itself also exited non-zero. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6662,7 +6760,7 @@ describe("duplex readiness gate replay-buffer reservation", () => { } function readyLine(): string { - return `${JSON.stringify({ version: 1, type: "ready", nonce: READY_NONCE })}\n`; + return `${JSON.stringify({ version: 2, type: "ready", nonce: READY_NONCE })}\n`; } it("charges the post-READY suffix and releases it after the broker handoff", async () => { diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 065a7f8c5b7..c21b7d3a874 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { Readable } from "node:stream"; import { randomBytes, randomUUID } from "node:crypto"; import type { SshRemoteExecutionSpec } from "./ssh.js"; import { @@ -59,6 +60,7 @@ import { DEFAULT_MAX_DUPLEX_FRAME_BYTES, type DuplexRequestFrame, } from "./duplex-frame-codec.js"; +import type { ReassembledBody } from "./duplex-body-spool.js"; import { createDuplexTelemetry, type DuplexFallbackReason, @@ -3132,9 +3134,24 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // file bridge and the duplex broker. The sandbox request carries only the // bridge token; the real agent token never leaves the host. const forwardBridgeRequest = async ( - request: { method: string; path: string; query: string; headers: Record; body: string }, + request: { + method: string; + path: string; + query: string; + headers: Record; + /** The file bridge passes the whole request body here as one string. */ + body?: string; + }, signal?: AbortSignal, - options?: { suppressDebugLog?: boolean }, + options?: { + suppressDebugLog?: boolean; + /** + * The duplex broker passes the reassembled request body here. The forward + * streams it to the host, so a spilled body never loads into memory on the + * receive side. The forward never disposes it; the broker owns its lifecycle. + */ + reassembledBody?: ReassembledBody; + }, ): Promise<{ status: number; headers: Record; body: string }> => { const method = request.method.trim().toUpperCase() || "GET"; // The per-request debug log prints the method, the path, and the query. The @@ -3159,12 +3176,26 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // the forward budget here, whichever comes first. const timeoutSignal = AbortSignal.timeout(forwardTimeoutMs); const forwardSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; - const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), { + // Build the request-body init. A GET or a HEAD carries no body. The duplex + // broker passes the reassembled body: stream it, so a body that spilled to + // disk never loads into memory here. A streamed body needs `duplex: "half"`. + // The file bridge passes the whole body as one string. + const forwardInit: RequestInit & { duplex?: "half" } = { method, headers, - ...(method === "GET" || method === "HEAD" ? {} : { body: request.body }), signal: forwardSignal, - }); + }; + if (method !== "GET" && method !== "HEAD") { + if (options?.reassembledBody) { + forwardInit.body = Readable.toWeb( + options.reassembledBody.createReadStream(), + ) as unknown as ReadableStream; + forwardInit.duplex = "half"; + } else if (typeof request.body === "string") { + forwardInit.body = request.body; + } + } + const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), forwardInit); if (emitDebugLog) { await onLog( "stdout", @@ -3375,14 +3406,14 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // token only now, after readiness passed. let broker: DuplexBridgeBroker | null = null; try { - broker = createDuplexBridgeBroker({ + broker = await createDuplexBridgeBroker({ channel: gate.brokerChannel, // Derive the nested budgets from the forward budget, so any forward // budget keeps the nested order the broker asserts at construction. budgets: deriveNestedDuplexBrokerBudgets(forwardTimeoutMs), forwardRequest: async ( request: DuplexRequestFrame, - options: { signal: AbortSignal }, + options: { signal: AbortSignal; body: ReassembledBody }, ): Promise => { const denialReason = authorizeSandboxCallbackBridgeRequestWithRoutes(request); if (denialReason) { @@ -3402,9 +3433,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { headers: sanitizeSandboxCallbackBridgeHeaders(request.headers), }; // Suppress the per-request debug log on the duplex path, so no route - // or query rides a log line here. + // or query rides a log line here. Stream the reassembled request + // body, so a spilled body never loads into memory on the host. return forwardBridgeRequest(sanitizedRequest, options.signal, { suppressDebugLog: true, + reassembledBody: options.body, }); }, // The duplex path emits only the fixed transport telemetry. It passes no diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index 9c3d073919f..5d8907ce048 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -1763,7 +1763,8 @@ export async function startSandboxCallbackBridgeServer(input: { * compatible with the host codec. {@link getSandboxDuplexGatewayCodecSource} * returns this exact source so a test can run every fixture vector against it. */ -const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 1; +const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 2; +const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024; const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 1000000; const DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES = 256; const DEFAULT_MAX_DUPLEX_DECODER_BYTES = ${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}; @@ -1792,6 +1793,10 @@ function duplexIsStringRecord(value) { return true; } +function duplexIsSafeNonNegativeInteger(value) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + function encodeDuplexFrame(frame) { return JSON.stringify(frame) + "\\n"; } @@ -1828,6 +1833,8 @@ function duplexValidateFrame(frame) { return duplexValidateRequest(frame); case "response": return duplexValidateResponse(frame); + case "body_chunk": + return duplexValidateBodyChunk(frame); case "ready": return duplexValidateReady(frame); case "heartbeat": @@ -1847,7 +1854,7 @@ function duplexValidateRequest(frame) { typeof frame.method !== "string" || typeof frame.path !== "string" || typeof frame.query !== "string" || - typeof frame.body !== "string" || + !duplexIsSafeNonNegativeInteger(frame.bodyByteCount) || !duplexIsStringRecord(frame.headers) ) { return duplexFail("malformed_frame", "request frame has a missing or wrong-typed field"); @@ -1862,7 +1869,7 @@ function duplexValidateResponse(frame) { if ( typeof frame.id !== "string" || typeof frame.status !== "number" || - typeof frame.body !== "string" || + !duplexIsSafeNonNegativeInteger(frame.bodyByteCount) || !duplexIsStringRecord(frame.headers) || typeof frame.outcome !== "string" || !DUPLEX_RESPONSE_OUTCOMES.has(frame.outcome) @@ -1875,6 +1882,19 @@ function duplexValidateResponse(frame) { return duplexOk(frame); } +function duplexValidateBodyChunk(frame) { + if (typeof frame.id !== "string" || typeof frame.data !== "string") { + return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed field"); + } + if (!duplexIsSafeNonNegativeInteger(frame.seq)) { + return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq"); + } + if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { + return duplexFail("id_too_large", "body_chunk frame id exceeds the maximum size"); + } + return duplexOk(frame); +} + function duplexValidateReady(frame) { if (typeof frame.nonce !== "string") { return duplexFail("malformed_frame", "ready frame has a missing or wrong-typed nonce"); @@ -2198,10 +2218,37 @@ async function runFileGateway() { }); } +// Split one whole body buffer into body_chunk frames. The gateway buffers the +// whole source body once, then splits it here. Each frame carries one fixed raw +// slice as base64 text, except the final frame, which carries the remaining +// bytes. A zero-length body yields no frame. Send-side true streaming is a +// separate goal; this whole-body split is acceptable for this gateway. +function splitDuplexBodyIntoChunks(id, body) { + const frames = []; + let seq = 0; + for (let offset = 0; offset < body.length; offset += DUPLEX_BODY_CHUNK_RAW_BYTES) { + const slice = body.subarray(offset, offset + DUPLEX_BODY_CHUNK_RAW_BYTES); + frames.push({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id: id, + seq: seq, + data: slice.toString("base64"), + }); + seq += 1; + } + return frames; +} + function runDuplexGateway() { // One outstanding local request per id. Each entry holds the HTTP resolver and // the wait-budget timer. const pending = new Map(); + // One in-flight response reassembly per id. The host returns a response as an + // envelope frame that carries bodyByteCount, then the body_chunk frames that + // carry the body. The gateway reassembles the response body in memory here; it + // does not spill, because a production response body stays small. + const responseAssembly = new Map(); let unavailable = false; let lossTriggered = false; let lastInboundAt = Date.now(); @@ -2243,27 +2290,102 @@ function runDuplexGateway() { }); } pending.clear(); + responseAssembly.clear(); const exitTimer = setTimeout(() => process.exit(0), lossExitGraceMs); if (typeof exitTimer.unref === "function") exitTimer.unref(); } + // Fail one outstanding request with a bounded local 502. The gateway calls it + // when a response reassembly breaks: a reordered seq, a base64 that is not + // canonical, or a total that overruns the declared body size. The host is the + // response peer, so this is a defensive local error, not a channel loss. + function failRequest(id, message) { + responseAssembly.delete(id); + const entry = pending.get(id); + if (!entry) return; + pending.delete(id); + clearTimeout(entry.timer); + entry.resolve({ + status: 502, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ error: message }), + }); + } + function handleInboundFrame(frame) { if (frame.type === "response") { const entry = pending.get(frame.id); if (!entry) return; - pending.delete(frame.id); - clearTimeout(entry.timer); // Map an indeterminate outcome to a non-retryable 409, the same contract // the file gateway applies through the outcome header. const statusCode = frame.outcome === "indeterminate" ? 409 : typeof frame.status === "number" ? frame.status : 200; - entry.resolve({ + const headers = frame.headers || {}; + if (frame.bodyByteCount === 0) { + // A zero-length body accepts no body_chunk, so the response completes now. + pending.delete(frame.id); + responseAssembly.delete(frame.id); + clearTimeout(entry.timer); + entry.resolve({ status: statusCode, headers: headers, body: "" }); + return; + } + // The body rides body_chunk frames that share this id. Record the envelope + // and wait for the chunks. + responseAssembly.set(frame.id, { status: statusCode, - headers: frame.headers || {}, - body: typeof frame.body === "string" ? frame.body : "", + headers: headers, + bodyByteCount: frame.bodyByteCount, + received: 0, + nextSeq: 0, + chunks: [], }); return; } + if (frame.type === "body_chunk") { + const asm = responseAssembly.get(frame.id); + if (!asm) return; + if (frame.seq !== asm.nextSeq) { + failRequest(frame.id, "duplex response body_chunk seq is out of order"); + return; + } + const decoded = Buffer.from(frame.data, "base64"); + if (decoded.toString("base64") !== frame.data) { + failRequest(frame.id, "duplex response body_chunk is not canonical base64"); + return; + } + if (decoded.length === 0) { + failRequest(frame.id, "duplex response body_chunk is empty"); + return; + } + if (asm.received + decoded.length > asm.bodyByteCount) { + failRequest(frame.id, "duplex response body overruns the declared size"); + return; + } + const isFinal = asm.received + decoded.length === asm.bodyByteCount; + if ( + decoded.length > DUPLEX_BODY_CHUNK_RAW_BYTES || + (!isFinal && decoded.length !== DUPLEX_BODY_CHUNK_RAW_BYTES) + ) { + failRequest(frame.id, "duplex response body_chunk has the wrong size"); + return; + } + asm.nextSeq += 1; + asm.received += decoded.length; + asm.chunks.push(decoded); + if (asm.received === asm.bodyByteCount) { + const entry = pending.get(frame.id); + responseAssembly.delete(frame.id); + if (!entry) return; + pending.delete(frame.id); + clearTimeout(entry.timer); + entry.resolve({ + status: asm.status, + headers: asm.headers, + body: Buffer.concat(asm.chunks).toString("utf8"), + }); + } + return; + } if (frame.type === "close") { triggerLoss("host sent a close frame"); return; @@ -2320,11 +2442,14 @@ function runDuplexGateway() { return; } const requestId = randomUUID(); - const requestBody = await readBody(req); + const requestBodyBuffer = Buffer.from(await readBody(req), "utf8"); if (unavailable) { writeJsonResponse(res, 503, { error: "bridge_unavailable" }); return; } + // The request body rides body_chunk frames. The envelope carries only the + // raw byte count, so the envelope stays small and the body splits into + // fixed-size slices that each stay under the frame bound. const requestFrame = { version: DUPLEX_FRAME_VERSION, type: "request", @@ -2333,20 +2458,36 @@ function runDuplexGateway() { path: url.pathname, query: url.search, headers: normalizeHeaders(req.headers), - body: requestBody, + bodyByteCount: requestBodyBuffer.length, }; - // Enforce the frame size bound on encode. When the request body makes the - // frame exceed the bound, fail this one local request with a clean 413. Do - // not write the frame and do not trigger loss. The frame never leaves the - // gateway, so no other in-flight request is affected and the channel stays - // open. const encodedRequest = encodeDuplexFrameChecked(requestFrame); if (!encodedRequest.ok) { writeJsonResponse(res, 413, { error: "request_too_large" }); return; } + // Pre-encode every body_chunk frame and enforce the frame size bound on + // each. A fixed raw slice never exceeds the bound, so this guard is + // defensive. On a rejection, fail this one local request with a clean 413. + // The frames never leave the gateway, so no other in-flight request is + // affected and the channel stays open. + const chunkFrames = splitDuplexBodyIntoChunks(requestId, requestBodyBuffer); + const encodedChunks = []; + let chunkTooLarge = false; + for (const chunk of chunkFrames) { + const encodedChunk = encodeDuplexFrameChecked(chunk); + if (!encodedChunk.ok) { + chunkTooLarge = true; + break; + } + encodedChunks.push(encodedChunk.line); + } + if (chunkTooLarge) { + writeJsonResponse(res, 413, { error: "request_too_large" }); + return; + } const response = await new Promise((resolve) => { const timer = setTimeout(() => { + responseAssembly.delete(requestId); if (pending.delete(requestId)) { resolve({ status: 502, @@ -2357,6 +2498,7 @@ function runDuplexGateway() { }, responseTimeoutMs); pending.set(requestId, { resolve: resolve, timer: timer }); process.stdout.write(encodedRequest.line); + for (const line of encodedChunks) process.stdout.write(line); }); res.statusCode = typeof response.status === "number" ? response.status : 200; for (const [key, value] of Object.entries(response.headers || {})) { From 63df7ad2b3b26e3684e322d421d767f3f107635e Mon Sep 17 00:00:00 2001 From: Nicky Leach Date: Sun, 23 Aug 2026 09:44:59 -0700 Subject: [PATCH 6/6] feat(login): use the login pseudo-terminal for Codex device login and de-Claude the shared channel (#12020) ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Agent adapters use provider-specific login flows > - Codex device login needs a live pseudo-terminal (PTY), while the shared channel still uses Claude-specific names > - The old streamed-exec path does not provide the prompt transport that Codex needs > - This pull request moves Codex device login to the shared login PTY and removes the dead streamed-exec path > - The benefit is one controlled login transport with fail-closed capability checks and safer credential reads ## Linked Issues or Issue Description **Problem or motivation** Codex device login used a streamed-exec path that did not provide the required prompt transport. The shared login channel also exposed Claude-specific names outside Claude code. **Expected behavior** The host selects a fixed login command from trusted adapter data. Codex login uses the provider login PTY. Providers without that capability fail closed. **Proposed solution** Use a server-controlled session home, create and validate it as a fresh 0700 directory, read credentials from one validated descriptor, and rename shared channel names to the neutral login PTY family. **Alternatives considered** Keep the shared login PTY as the single transport. Do not keep the removed streamed-exec path because it cannot provide the required prompt transport. **Roadmap alignment** This change supports the planned login transport work. It does not add a separate roadmap item. ## What Changed - Route Codex device login through the shared login PTY transport. - Select the login command from a closed internal command key. - Carry a server-controlled session home through the launch contract. - Create and validate the session home as a fresh 0700 directory owned by the login user. - Read the credential file with descriptor-relative, no-follow path walking and final descriptor checks. - Gate the login route and run lease on the provider login PTY capability. - Rename shared channel names to the neutral login PTY family. - Remove the streamed-exec transport value, selector field, driver branch, and related tests. - Hide Codex login in the user interface when the provider lacks the login PTY capability. ## Verification - Server unit suites pass: 89/89. - Adapter-utils suites pass: 262/262. - Codex-local suites pass: 326/326. - Credential-read reader suite passes: 20/20. - Daytona login PTY suite passes: 30/30. - Device-login suites pass: 56/56. - TypeScript checks pass for server, adapter-utils, and UI. - GitHub Actions must pass after pull request creation. - Greptile review must reach 5/5 with no open P2 findings, recommendations, or follow-ups. ## Risks - Providers without a login PTY capability lose Codex login support by design. - The credential read rejects invalid ownership, mode, type, path, and size. - The launch-time sandbox directory race remains outside the threat model because the login runs inside the sandbox and a hostile sandbox already controls its credential. ## Model Used OpenAI Codex, GPT-5, tool use and code review assistance. The exact context window and reasoning mode are not exposed by the runtime. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- packages/adapter-utils/src/index.ts | 2 - .../src/login-capability.test.ts | 7 - .../adapter-utils/src/login-capability.ts | 15 - ...rt.test.ts => login-pty-transport.test.ts} | 28 +- ...en-transport.ts => login-pty-transport.ts} | 22 +- .../src/server/device-login-parse.test.ts | 56 ++ .../src/server/device-login-runner.test.ts | 4 +- .../src/server/device-login-runner.ts | 35 +- .../sandbox-providers/daytona/package.json | 2 +- .../daytona/src/duplex-command-stream.ts | 6 +- .../daytona/src/login-pty.test.ts | 645 ++++++++++++++++++ .../daytona/src/login-pty.ts | 586 ++++++++++++++++ .../sandbox-providers/daytona/src/manifest.ts | 6 +- .../sandbox-providers/daytona/src/plugin.ts | 94 +-- .../daytona/src/scripts/login-home-create.cjs | 69 ++ .../src/scripts/login-home-inspect.cjs | 66 ++ .../daytona/src/setup-token-pty.test.ts | 222 ------ .../daytona/src/setup-token-pty.ts | 207 ------ packages/plugins/sdk/src/define-plugin.ts | 32 +- packages/plugins/sdk/src/index.ts | 4 +- packages/plugins/sdk/src/protocol.ts | 75 +- packages/plugins/sdk/src/testing.ts | 2 +- packages/plugins/sdk/src/types.ts | 8 +- packages/plugins/sdk/src/worker-rpc-host.ts | 74 +- .../plugins/sdk/tests/worker-rpc-host.test.ts | 51 +- server/package.json | 2 +- .../agent-device-login-routes.test.ts | 116 +++- ...codex-device-login-credential-read.test.ts | 307 +++++++++ .../codex-device-login-service.test.ts | 175 +++-- .../codex-login-session-runtime.test.ts | 177 +++++ ...en-pty.cjs => plugin-worker-login-pty.cjs} | 22 +- .../__tests__/plugin-worker-manager.test.ts | 125 ++-- server/src/adapters/plugin-loader.test.ts | 1 - server/src/adapters/registry.test.ts | 2 - server/src/adapters/registry.ts | 11 +- server/src/app.ts | 4 +- server/src/routes/adapters.test.ts | 4 - server/src/routes/adapters.ts | 10 +- server/src/routes/agents.ts | 125 +++- server/src/routes/setup-token-route.test.ts | 4 +- .../codex-device-login-credential-read.ts | 144 ++++ .../services/codex-device-login-service.ts | 251 +++++-- server/src/services/login-command.test.ts | 88 +++ server/src/services/login-command.ts | 105 +++ server/src/services/plugin-worker-manager.ts | 208 +++--- .../src/services/scripts/codex-auth-read.cjs | 93 +++ .../setup-token-transport-binding.test.ts | 84 ++- .../services/setup-token-transport-binding.ts | 73 +- ui/src/api/adapters.ts | 1 - .../AgentConfigForm.render.test.tsx | 55 +- ui/src/components/AgentConfigForm.tsx | 22 +- ui/src/pages/NewAgent.test.tsx | 6 +- 52 files changed, 3500 insertions(+), 1033 deletions(-) rename packages/adapter-utils/src/{setup-token-transport.test.ts => login-pty-transport.test.ts} (88%) rename packages/adapter-utils/src/{setup-token-transport.ts => login-pty-transport.ts} (89%) create mode 100644 packages/plugins/sandbox-providers/daytona/src/login-pty.test.ts create mode 100644 packages/plugins/sandbox-providers/daytona/src/login-pty.ts create mode 100644 packages/plugins/sandbox-providers/daytona/src/scripts/login-home-create.cjs create mode 100644 packages/plugins/sandbox-providers/daytona/src/scripts/login-home-inspect.cjs delete mode 100644 packages/plugins/sandbox-providers/daytona/src/setup-token-pty.test.ts delete mode 100644 packages/plugins/sandbox-providers/daytona/src/setup-token-pty.ts create mode 100644 server/src/__tests__/codex-device-login-credential-read.test.ts create mode 100644 server/src/__tests__/codex-login-session-runtime.test.ts rename server/src/__tests__/fixtures/{plugin-worker-setup-token-pty.cjs => plugin-worker-login-pty.cjs} (92%) create mode 100644 server/src/services/codex-device-login-credential-read.ts create mode 100644 server/src/services/login-command.test.ts create mode 100644 server/src/services/login-command.ts create mode 100644 server/src/services/scripts/codex-auth-read.cjs diff --git a/packages/adapter-utils/src/index.ts b/packages/adapter-utils/src/index.ts index aa7bb230299..283097815d4 100644 --- a/packages/adapter-utils/src/index.ts +++ b/packages/adapter-utils/src/index.ts @@ -86,7 +86,6 @@ export type { export { inferOpenAiCompatibleBiller } from "./billing.js"; export { ADAPTER_LOGIN_PANEL_MODES, - ADAPTER_LOGIN_SANDBOX_TRANSPORTS, ADAPTER_LOGIN_TIMEOUT_POLICIES, ADAPTER_LOGIN_COMPLETION_CLAIMS, assertValidAdapterLoginCapability, @@ -94,7 +93,6 @@ export { } from "./login-capability.js"; export type { AdapterLoginPanelMode, - AdapterLoginSandboxTransport, AdapterLoginTimeoutPolicy, AdapterLoginCompletionClaim, AdapterLoginPrompt, diff --git a/packages/adapter-utils/src/login-capability.test.ts b/packages/adapter-utils/src/login-capability.test.ts index a79a4e2dfa1..617103defce 100644 --- a/packages/adapter-utils/src/login-capability.test.ts +++ b/packages/adapter-utils/src/login-capability.test.ts @@ -10,7 +10,6 @@ import { function validCapability(): AdapterLoginCapability { return { panelMode: "displayed_code", - sandboxTransport: "streamed_exec", timeoutPolicy: "caller_bounded", getCommand: () => "vendor login", parsePrompt: () => null, @@ -25,7 +24,6 @@ describe("assertValidAdapterLoginCapability", () => { it("accepts a well-formed capability with every optional member set", () => { const capability: AdapterLoginCapability = { panelMode: "submitted_browser_code", - sandboxTransport: "pseudo_terminal", timeoutPolicy: "fixed", getCommand: () => "vendor setup-token", parsePrompt: () => null, @@ -48,11 +46,6 @@ describe("assertValidAdapterLoginCapability", () => { expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/panelMode/); }); - it("rejects an unknown sandboxTransport", () => { - const bad = { ...validCapability(), sandboxTransport: "http" }; - expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/sandboxTransport/); - }); - it("rejects an unknown timeoutPolicy", () => { const bad = { ...validCapability(), timeoutPolicy: "unbounded" }; expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/timeoutPolicy/); diff --git a/packages/adapter-utils/src/login-capability.ts b/packages/adapter-utils/src/login-capability.ts index 986dc8a6b6e..9eac8e13466 100644 --- a/packages/adapter-utils/src/login-capability.ts +++ b/packages/adapter-utils/src/login-capability.ts @@ -18,14 +18,6 @@ export const ADAPTER_LOGIN_PANEL_MODES = ["displayed_code", "submitted_browser_code"] as const; export type AdapterLoginPanelMode = (typeof ADAPTER_LOGIN_PANEL_MODES)[number]; -/** - * The sandbox transport that the login command needs. `streamed_exec` runs the - * command over the streamed exec channel. `pseudo_terminal` runs the command on - * a real pseudo-terminal, because a pipe emits no login prompt. - */ -export const ADAPTER_LOGIN_SANDBOX_TRANSPORTS = ["streamed_exec", "pseudo_terminal"] as const; -export type AdapterLoginSandboxTransport = (typeof ADAPTER_LOGIN_SANDBOX_TRANSPORTS)[number]; - /** * The host-side timeout policy. `caller_bounded` lets the caller set the * timeout. `fixed` binds the timeout to a fixed adapter value. @@ -68,8 +60,6 @@ export interface AdapterLoginCompletionContext { export interface AdapterLoginCapability { /** The login panel mode. */ panelMode: AdapterLoginPanelMode; - /** The sandbox transport that the login command needs. */ - sandboxTransport: AdapterLoginSandboxTransport; /** The host-side timeout policy. */ timeoutPolicy: AdapterLoginTimeoutPolicy; /** Returns the fixed, non-secret login command. */ @@ -122,11 +112,6 @@ export function assertValidAdapterLoginCapability( `${prefix}: "panelMode" must be one of ${ADAPTER_LOGIN_PANEL_MODES.join(", ")}.`, ); } - if (!isOneOf(ADAPTER_LOGIN_SANDBOX_TRANSPORTS, cap.sandboxTransport)) { - throw new Error( - `${prefix}: "sandboxTransport" must be one of ${ADAPTER_LOGIN_SANDBOX_TRANSPORTS.join(", ")}.`, - ); - } if (!isOneOf(ADAPTER_LOGIN_TIMEOUT_POLICIES, cap.timeoutPolicy)) { throw new Error( `${prefix}: "timeoutPolicy" must be one of ${ADAPTER_LOGIN_TIMEOUT_POLICIES.join(", ")}.`, diff --git a/packages/adapter-utils/src/setup-token-transport.test.ts b/packages/adapter-utils/src/login-pty-transport.test.ts similarity index 88% rename from packages/adapter-utils/src/setup-token-transport.test.ts rename to packages/adapter-utils/src/login-pty-transport.test.ts index 3644e35581e..d73bddcb166 100644 --- a/packages/adapter-utils/src/setup-token-transport.test.ts +++ b/packages/adapter-utils/src/login-pty-transport.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "vitest"; import { - createSetupTokenPtyTransport, - type SetupTokenPtySession, -} from "./setup-token-transport.js"; + createLoginPtyTransport, + type LoginPtySession, +} from "./login-pty-transport.js"; // The Enter byte the terminal login UI reads to submit the browser code. The // login runner appends this byte to the code. The transport forwards the bytes @@ -14,7 +14,7 @@ const ENTER = "\r"; * stream on demand, and records the stop and the close. The tests use it in * place of a real pseudo-terminal, so the transport runs with no sandbox. */ -function createFakePtySession(): SetupTokenPtySession & { +function createFakePtySession(): LoginPtySession & { writes: string[]; emit: (chunk: string) => void; finish: (exitCode: number | null) => void; @@ -74,10 +74,10 @@ function createFakePtySession(): SetupTokenPtySession & { }; } -describe("createSetupTokenPtyTransport", () => { +describe("createLoginPtyTransport", () => { it("delivers delayed input to the process", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); await session.ready; @@ -94,7 +94,7 @@ describe("createSetupTokenPtyTransport", () => { it("returns incremental terminal output to the runner", async () => { const session = createFakePtySession(); const received: string[] = []; - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", (chunk) => { received.push(chunk); @@ -114,7 +114,7 @@ describe("createSetupTokenPtyTransport", () => { it("delivers the Enter byte after the browser code", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); await session.ready; @@ -133,7 +133,7 @@ describe("createSetupTokenPtyTransport", () => { it("resolves start with the child exit code", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); session.finish(7); @@ -143,7 +143,7 @@ describe("createSetupTokenPtyTransport", () => { it("stops the child with a direct kill", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); await session.ready; @@ -156,7 +156,7 @@ describe("createSetupTokenPtyTransport", () => { it("disposes the session resources", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); session.finish(0); @@ -168,7 +168,7 @@ describe("createSetupTokenPtyTransport", () => { it("stays safe when stop and dispose run before start", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); // The runner may stop or dispose before it starts the child. The transport // must not throw, and it must not open a session for a run it already stopped. @@ -185,7 +185,7 @@ describe("createSetupTokenPtyTransport", () => { const openGate = new Promise((resolve) => { resolveOpen = resolve; }); - const transport = createSetupTokenPtyTransport(async () => { + const transport = createLoginPtyTransport(async () => { await openGate; return session; }); @@ -207,7 +207,7 @@ describe("createSetupTokenPtyTransport", () => { const openGate = new Promise((resolve) => { resolveOpen = resolve; }); - const transport = createSetupTokenPtyTransport(async () => { + const transport = createLoginPtyTransport(async () => { await openGate; return session; }); diff --git a/packages/adapter-utils/src/setup-token-transport.ts b/packages/adapter-utils/src/login-pty-transport.ts similarity index 89% rename from packages/adapter-utils/src/setup-token-transport.ts rename to packages/adapter-utils/src/login-pty-transport.ts index af8b0c856cc..48369bdbe49 100644 --- a/packages/adapter-utils/src/setup-token-transport.ts +++ b/packages/adapter-utils/src/login-pty-transport.ts @@ -1,4 +1,4 @@ -// The setup-token pseudo-terminal transport. It gives the Claude `setup-token` +// The login pseudo-terminal (PTY) transport. It gives the Claude `setup-token` // login runner a child that runs on a real pseudo-terminal (PTY). The command // needs a PTY: pipe stdio emits no login prompt. The transport starts the // command on a PTY, streams the incremental terminal output, delivers delayed @@ -7,7 +7,7 @@ // This module is provider-agnostic and pure. It holds no Node built-in import, // so the browser-safe root entry can re-export it. A sandbox provider (for // example the Daytona plugin) opens the concrete pseudo-terminal through a -// {@link SetupTokenPtySessionOpener}. A unit test opens a fake pseudo-terminal +// {@link LoginPtySessionOpener}. A unit test opens a fake pseudo-terminal // through the same opener. // // Boundary (ANSI and OSC 8): the transport forwards the raw terminal bytes @@ -20,7 +20,7 @@ * raw terminal output, accepts delayed input, and stops the child. The session * forwards the raw terminal bytes; it runs no ANSI or OSC 8 handling. */ -export interface SetupTokenPtySession { +export interface LoginPtySession { /** * Registers the one output listener. The session streams each raw terminal * output chunk to `listener`, in order, as the pseudo-terminal emits it. @@ -49,7 +49,7 @@ export interface SetupTokenPtySession { * Opens a pseudo-terminal session for `command`. A provider binds this to its * sandbox. The transport calls it one time, on start. */ -export type SetupTokenPtySessionOpener = (command: string) => Promise; +export type LoginPtySessionOpener = (command: string) => Promise; /** * The child side of the setup-token run, in the shape the login runner needs. @@ -60,7 +60,7 @@ export type SetupTokenPtySessionOpener = (command: string) => Promise { if (started) { - throw new Error("setup-token PTY transport already started."); + throw new Error("login PTY transport already started."); } started = true; const opened = await open(command); diff --git a/packages/adapters/codex-local/src/server/device-login-parse.test.ts b/packages/adapters/codex-local/src/server/device-login-parse.test.ts index cfd6fa9b981..8768d183321 100644 --- a/packages/adapters/codex-local/src/server/device-login-parse.test.ts +++ b/packages/adapters/codex-local/src/server/device-login-parse.test.ts @@ -42,6 +42,62 @@ describe("parseDeviceLoginPrompt", () => { expect(result?.code).toBe("ABCD-EFGHJ"); }); + it("parse_returns_url_and_code_from_carriage_return_line_endings", () => { + // A pseudo-terminal ends each line with a carriage return and a line feed. + // The carriage return is whitespace, so the URL token stops before it, and the + // code line trims it. The parser reads the tokens the same as line-feed output. + const text = [ + "1. Open this link in your browser and sign in to your account", + EXACT_URL, + "2. Enter this one-time code (expires in 15 minutes)", + "ABCD-EFGHJ", + ].join("\r\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_returns_url_and_code_with_a_terminal_title_osc_sequence", () => { + // A pseudo-terminal can emit an Operating System Command (OSC) title sequence + // in the stream. The device-auth prompt still prints the URL and the code as + // plain standalone tokens on their own lines. The OSC title sequence carries no + // URL token, so it does not shift the URL or the code, and the parser reads + // both tokens. + const osc = "\x1b]0;Codex\x07"; + const text = [ + `${osc}1. Open this link in your browser and sign in to your account`, + EXACT_URL, + "2. Enter this one-time code (expires in 15 minutes)", + "ABCD-EFGHJ", + ].join("\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_returns_url_and_code_from_a_merged_raw_pseudo_terminal_frame", () => { + // One buffer holds the whole prompt as a real pseudo-terminal emits it: a + // leading title Operating System Command (OSC) sequence, color Control + // Sequence Introducer (CSI) sequences around the URL and the code, and + // carriage-return line endings. The parser removes the color sequences and + // reads the plain tokens. + const title = "\x1b]0;Codex login\x07"; + const cyan = "\x1b[36m"; + const bold = "\x1b[1m"; + const reset = "\x1b[0m"; + const text = + `${title}1. Open this link in your browser and sign in to your account\r\n` + + `${cyan}${EXACT_URL}${reset}\r\n` + + `2. Enter this one-time code (expires in 15 minutes)\r\n` + + `${bold}ABCD-EFGHJ${reset}\r\n`; + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + it("parse_returns_null_when_prompt_absent", () => { const text = "Some unrelated log line\nNothing to see here\n"; expect(parseDeviceLoginPrompt(text)).toBeNull(); diff --git a/packages/adapters/codex-local/src/server/device-login-runner.test.ts b/packages/adapters/codex-local/src/server/device-login-runner.test.ts index 7a149581499..b9393e8bbad 100644 --- a/packages/adapters/codex-local/src/server/device-login-runner.test.ts +++ b/packages/adapters/codex-local/src/server/device-login-runner.test.ts @@ -24,10 +24,10 @@ interface FakeDriverOptions { function createFakeDriver(options: FakeDriverOptions = {}) { const disposeCalls = { count: 0 }; const driver: SandboxLoginDriver = { - async execStreaming(_command, onStdout) { + async start(_command, onData) { if (options.execError) throw options.execError; for (const chunk of options.chunks ?? []) { - onStdout(chunk); + onData(chunk); } if (options.hang) { // Never resolve. The runner must fall back to its timeout or signal. diff --git a/packages/adapters/codex-local/src/server/device-login-runner.ts b/packages/adapters/codex-local/src/server/device-login-runner.ts index 4c66a702c0e..968031e1130 100644 --- a/packages/adapters/codex-local/src/server/device-login-runner.ts +++ b/packages/adapters/codex-local/src/server/device-login-runner.ts @@ -38,18 +38,33 @@ const MAX_PARSE_BUFFER_CHARS = 64 * 1024; /** * The sandbox side of the device-login run. The runner never calls Daytona - * directly; a caller injects a concrete driver. A production driver binds these - * three methods to a non-persisting Daytona exec path, a file read, and a - * sandbox delete. + * directly; a caller injects a concrete driver. A production driver binds `start` + * to the shared login pseudo-terminal (PTY) transport, binds `readFile` to a + * descriptor-bound credential read, and binds `dispose` to the transport dispose. + * + * The `start` shape matches the shared login pseudo-terminal transport start + * method, so the driver runs the login command on a real pseudo-terminal. The + * login command + * needs a pseudo-terminal: pipe stdio emits no login prompt. The runner never + * calls `stop` or `write` on the driver; the device-login flow needs no delayed + * input, and the service owns the sandbox delete. The runner disposes the driver + * one time on every terminal state. */ export interface SandboxLoginDriver extends LoginRunnerDisposable { /** - * Runs `command` in the sandbox and streams standard output to `onStdout` in - * memory. Resolves with the command exit code when the command ends. A driver - * must not persist the raw output to any durable log. + * Starts `command` on a pseudo-terminal and streams the terminal output to + * `onData` in memory, in order, as the pseudo-terminal emits it. Resolves with + * the command exit code when the command ends. A driver must not persist the raw + * output to any durable log. + */ + start(command: string, onData: (chunk: string) => void): Promise<{ exitCode: number | null }>; + /** + * Reads the credential bytes with one descriptor-bound read. The read is + * separate from the pseudo-terminal session; the session has no file-read + * method. A driver runs one fixed, server-controlled operation that opens the + * verified session home and the credential file with no symlink follow, checks + * the opened descriptor, and reads only from that same descriptor. */ - execStreaming(command: string, onStdout: (chunk: string) => void): Promise<{ exitCode: number | null }>; - /** Reads the bytes of one file from the sandbox. */ readFile(path: string): Promise; } @@ -123,8 +138,8 @@ export async function runDeviceLogin( return { outcome: "cancelled", exitCode: null, promptSurfaced }; } - const exec = driver.execStreaming(command, onStdout); - const raced = await raceLoginRunnerExit(exec, timeoutMs, signal); + const started = driver.start(command, onStdout); + const raced = await raceLoginRunnerExit(started, timeoutMs, signal); if (raced.kind === "timeout") { log("[paperclip] Device login timed out; disposing the sandbox."); diff --git a/packages/plugins/sandbox-providers/daytona/package.json b/packages/plugins/sandbox-providers/daytona/package.json index ddc5acd934f..666300e93cc 100644 --- a/packages/plugins/sandbox-providers/daytona/package.json +++ b/packages/plugins/sandbox-providers/daytona/package.json @@ -42,7 +42,7 @@ ], "scripts": { "prebuild": "pnpm -C ../../../.. --filter @paperclipai/plugin-sdk ensure-build-deps", - "build": "rm -rf dist && tsc", + "build": "rm -rf dist && tsc && mkdir -p dist/scripts && cp -R src/scripts/. dist/scripts/", "clean": "rm -rf dist", "typecheck": "pnpm -C ../../../.. --filter @paperclipai/plugin-sdk ensure-build-deps && tsc --noEmit", "test": "vitest run --config vitest.config.ts", diff --git a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts index 4507ba774e8..3da94be8fc1 100644 --- a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts +++ b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts @@ -27,7 +27,7 @@ // Dependency boundary: this provider plugin ships standalone (the workspace // excludes `packages/plugins/sandbox-providers/**`). So the module imports no // workspace package. It reuses the narrow Daytona PTY surface that -// `setup-token-pty.ts` declares (`DaytonaPtyProcess`, `DaytonaPtyHandle`, +// `login-pty.ts` declares (`DaytonaPtyProcess`, `DaytonaPtyHandle`, // `DaytonaPtyCreateOptions`), so a caller passes `sandbox.process` with no // adapter. The narrow surface keeps the module unit-testable with a fake PTY. @@ -37,13 +37,13 @@ import type { DaytonaPtyCreateOptions, DaytonaPtyHandle, DaytonaPtyProcess, -} from "./setup-token-pty.js"; +} from "./login-pty.js"; export type { DaytonaPtyCreateOptions, DaytonaPtyHandle, DaytonaPtyProcess, -} from "./setup-token-pty.js"; +} from "./login-pty.js"; import { sendPtyInputInChunks } from "./pty-chunked-input.js"; diff --git a/packages/plugins/sandbox-providers/daytona/src/login-pty.test.ts b/packages/plugins/sandbox-providers/daytona/src/login-pty.test.ts new file mode 100644 index 00000000000..398106dae33 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/login-pty.test.ts @@ -0,0 +1,645 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + composeLaunchLine, + createDaytonaLoginHomeFs, + createDaytonaLoginPtySessionOpener, + encodePosixShellArg, + openDaytonaLoginPtySession, + type DaytonaExecResult, + type DaytonaLoginHomeFs, + type DaytonaPtyCreateOptions, + type DaytonaPtyHandle, + type DaytonaPtyProcess, + type DaytonaSandboxExec, + type LoginHomeInspection, + type LoginPtyLaunchDescriptor, +} from "./login-pty.js"; + +// The Enter byte the terminal login UI reads to submit the browser code. +const ENTER = "\r"; + +const HOME = "/tmp/paperclip-adapter-login/11111111-2222-4333-8444-555555555555"; +const LOGIN_UID = 1000; + +const CLAUDE: LoginPtyLaunchDescriptor = { loginCommandKey: "claude", sessionHome: HOME }; +const CODEX: LoginPtyLaunchDescriptor = { loginCommandKey: "codex", sessionHome: HOME }; + +/** A directory entry in the virtual filesystem. */ +type FakeEntry = Omit; + +const ABSENT: LoginHomeInspection = { + exists: false, + isSymlink: false, + isDirectory: false, + mode: "", + ownerUid: null, +}; + +/** + * A fake login-home filesystem. It models one virtual path. `createDirectory` + * fails when the path exists and otherwise stores `createAs` (a clean 0700 + * directory owned by the login user by default). A test seeds a pre-existing + * entry, forces a bad created entry, or swaps the entry through `onCreatePty`, so + * a test drives each rejection and the pre-launch re-check. + */ +function createFakeHomeFs(config?: { + loginUid?: number; + seed?: FakeEntry; + createAs?: FakeEntry; + createShouldFail?: boolean; +}): DaytonaLoginHomeFs & { + created: Array<{ path: string; mode: string }>; + inspectCount: number; + setEntry: (entry: FakeEntry | null) => void; +} { + const loginUid = config?.loginUid ?? LOGIN_UID; + const cleanDir: FakeEntry = { + isSymlink: false, + isDirectory: true, + mode: "700", + ownerUid: loginUid, + }; + let entry: FakeEntry | null = config?.seed ?? null; + const created: Array<{ path: string; mode: string }> = []; + const state = { inspectCount: 0 }; + return { + created, + get inspectCount() { + return state.inspectCount; + }, + setEntry(next: FakeEntry | null): void { + entry = next; + }, + async loginUserId(): Promise { + return loginUid; + }, + async inspect(): Promise { + state.inspectCount += 1; + return entry ? { exists: true, ...entry } : ABSENT; + }, + async createDirectory(path: string, mode: string): Promise { + if (config?.createShouldFail || entry) { + throw new Error("LOGIN_PTY_HOME_REJECTED"); + } + created.push({ path, mode }); + entry = config?.createAs ?? cleanDir; + }, + }; +} + +/** + * A fake Daytona PTY handle. It records each input write, drives the output + * stream on demand, and records the kill and the disconnect. The tests use it in + * place of the real SDK `PtyHandle`, so the session runs with no sandbox. + */ +function createFakePtyHandle( + onData: (data: Uint8Array) => void | Promise, + onWaitForConnection?: () => void, +): DaytonaPtyHandle & { + inputs: string[]; + emitText: (text: string) => void; + emitBytes: (bytes: Uint8Array) => void; + finish: (exitCode: number | undefined) => void; + killed: number; + disconnected: number; +} { + const encoder = new TextEncoder(); + const inputs: string[] = []; + let resolveWait: ((value: { exitCode?: number; error?: string }) => void) | null = null; + const waitPromise = new Promise<{ exitCode?: number; error?: string }>((resolve) => { + resolveWait = resolve; + }); + let killed = 0; + let disconnected = 0; + return { + async waitForConnection(): Promise { + onWaitForConnection?.(); + }, + async sendInput(data: string | Uint8Array): Promise { + inputs.push(typeof data === "string" ? data : new TextDecoder().decode(data)); + }, + wait(): Promise<{ exitCode?: number; error?: string }> { + return waitPromise; + }, + async kill(): Promise { + killed += 1; + }, + async disconnect(): Promise { + disconnected += 1; + }, + emitText(text: string): void { + onData(encoder.encode(text)); + }, + emitBytes(bytes: Uint8Array): void { + onData(bytes); + }, + finish(exitCode: number | undefined): void { + resolveWait?.({ exitCode }); + }, + get inputs(): string[] { + return inputs; + }, + get killed(): number { + return killed; + }, + get disconnected(): number { + return disconnected; + }, + }; +} + +/** + * A fake Daytona process. It opens one fake PTY handle and records the create + * options, so a test asserts the terminal size and the launch line. `onCreatePty` + * runs when the session opens the terminal, so a test swaps a symlink in after + * the directory creation but before the launch input. + */ +function createFakeProcess(onCreatePty?: () => void): DaytonaPtyProcess & { + handle: ReturnType | null; + createOptions: DaytonaPtyCreateOptions | null; + createCount: number; +} { + const state: { + handle: ReturnType | null; + createOptions: DaytonaPtyCreateOptions | null; + createCount: number; + } = { handle: null, createOptions: null, createCount: 0 }; + return { + get handle() { + return state.handle; + }, + get createOptions() { + return state.createOptions; + }, + get createCount() { + return state.createCount; + }, + async createPty(options: DaytonaPtyCreateOptions): Promise { + state.createCount += 1; + state.createOptions = options; + onCreatePty?.(); + const handle = createFakePtyHandle(options.onData); + state.handle = handle; + return handle; + }, + }; +} + +describe("encodePosixShellArg", () => { + it("wraps a value in single quotes and neutralizes metacharacters", () => { + expect(encodePosixShellArg("/tmp/x")).toBe("'/tmp/x'"); + // A metacharacter, a space, and a semicolon stay literal inside the quotes. + expect(encodePosixShellArg("; rm -rf / #")).toBe("'; rm -rf / #'"); + // An embedded single quote closes, escapes, and reopens the quote. + expect(encodePosixShellArg("a'b")).toBe("'a'\\''b'"); + expect(encodePosixShellArg("$(id)")).toBe("'$(id)'"); + }); +}); + +describe("composeLaunchLine", () => { + it("composes the Claude line with no CODEX_HOME", () => { + const line = composeLaunchLine(CLAUDE); + expect(line).toBe("exec claude setup-token"); + expect(line).not.toContain("CODEX_HOME"); + }); + + it("composes the Codex line with exactly one encoded CODEX_HOME", () => { + const line = composeLaunchLine(CODEX); + expect(line).toBe(`exec env CODEX_HOME='${HOME}' codex login --device-auth`); + // Exactly one CODEX_HOME assignment, and the exact approved Codex command. + expect(line.match(/CODEX_HOME=/g)).toHaveLength(1); + expect(line.endsWith("codex login --device-auth")).toBe(true); + }); +}); + +describe("openDaytonaLoginPtySession — session home", () => { + it("creates the exact UUID directory as a new 0700 directory owned by the login user", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await openDaytonaLoginPtySession(process, fs, CLAUDE); + + // The provider created the exact directory as a new 0700 directory. + expect(fs.created).toEqual([{ path: HOME, mode: "700" }]); + // The terminal opened and the launch line ran. + expect(process.createCount).toBe(1); + expect(process.handle?.inputs[0]).toBe("exec claude setup-token" + ENTER); + }); + + it("sends the Codex launch line with the encoded CODEX_HOME", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await openDaytonaLoginPtySession(process, fs, CODEX); + + expect(process.handle?.inputs[0]).toBe( + `exec env CODEX_HOME='${HOME}' codex login --device-auth` + ENTER, + ); + // The Claude line carries no CODEX_HOME; the Codex line carries exactly one. + expect((process.handle?.inputs[0]?.match(/CODEX_HOME=/g) ?? []).length).toBe(1); + }); + + it("rejects a pre-existing target before it creates the directory", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + seed: { isSymlink: false, isDirectory: true, mode: "700", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + // The provider never created the directory and never opened the terminal. + expect(fs.created).toEqual([]); + expect(process.createCount).toBe(0); + }); + + it("rejects a pre-existing symlink at the target", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + seed: { isSymlink: true, isDirectory: false, mode: "777", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created target that is a symlink", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: true, isDirectory: false, mode: "700", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created target that is not a directory", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: false, isDirectory: false, mode: "700", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created directory with a wrong owner", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: false, isDirectory: true, mode: "700", ownerUid: LOGIN_UID + 1 }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created directory with a wrong mode", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: false, isDirectory: true, mode: "755", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a symlink swapped in after creation, before the launch input", async () => { + // The directory validates cleanly, then a symlink replaces it while the + // terminal opens. The no-symlink re-check before the launch input fails. + const fs = createFakeHomeFs(); + const process = createFakeProcess(() => { + fs.setEntry({ isSymlink: true, isDirectory: false, mode: "777", ownerUid: LOGIN_UID }); + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + // The terminal opened, but the launch input never ran. + expect(process.createCount).toBe(1); + expect(process.handle?.inputs).toEqual([]); + }); + + it("rejects a descriptor with a command key outside the closed set", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await expect( + openDaytonaLoginPtySession(process, fs, { + loginCommandKey: "gemini" as unknown as "claude", + sessionHome: HOME, + }), + ).rejects.toThrow("LOGIN_PTY_DESCRIPTOR_REJECTED"); + // The provider never touched the filesystem or the terminal. + expect(fs.created).toEqual([]); + expect(process.createCount).toBe(0); + }); + + it("rejects a descriptor whose session home shape is wrong", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await expect( + openDaytonaLoginPtySession(process, fs, { + loginCommandKey: "codex", + sessionHome: "/tmp/paperclip-adapter-login/../etc", + }), + ).rejects.toThrow("LOGIN_PTY_DESCRIPTOR_REJECTED"); + expect(process.createCount).toBe(0); + }); +}); + +describe("openDaytonaLoginPtySession — session mechanics", () => { + it("starts the login command on a pseudo-terminal with a fixed size", async () => { + const process = createFakeProcess(); + + await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + + expect(process.createOptions?.cols).toBe(120); + expect(process.createOptions?.rows).toBe(30); + expect(process.handle?.inputs[0]).toBe("exec claude setup-token" + ENTER); + }); + + it("returns incremental terminal output to the listener", async () => { + const process = createFakeProcess(); + const received: string[] = []; + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.onData((chunk) => received.push(chunk)); + + process.handle?.emitText("the url below to sign in\n"); + process.handle?.emitText("Paste code here if prompted\n"); + + expect(received).toEqual(["the url below to sign in\n", "Paste code here if prompted\n"]); + }); + + it("buffers early output until the listener registers", async () => { + const process = createFakeProcess(); + const received: string[] = []; + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + process.handle?.emitText("early output "); + process.handle?.emitText("more output"); + expect(received).toEqual([]); + + session.onData((chunk) => received.push(chunk)); + expect(received).toEqual(["early output more output"]); + }); + + it("delivers the browser code plus the Enter byte to the command", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.write("BROWSERCODE" + ENTER); + // The write runs through the serialized chunk chain, so it lands on a later + // microtask. Flush the chain before the assertion. + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(process.handle?.inputs[1]).toBe("BROWSERCODE" + ENTER); + expect(process.handle?.inputs[1]?.endsWith(ENTER)).toBe(true); + }); + + it("keeps a multibyte character whole across two output chunks", async () => { + const process = createFakeProcess(); + const received: string[] = []; + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.onData((chunk) => received.push(chunk)); + + const euro = new TextEncoder().encode("€"); + process.handle?.emitBytes(euro.subarray(0, 2)); + process.handle?.emitBytes(euro.subarray(2)); + + expect(received.join("")).toBe("€"); + }); + + it("resolves wait with the command exit code", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + process.handle?.finish(9); + + await expect(session.wait()).resolves.toEqual({ exitCode: 9 }); + }); + + it("maps an absent exit code to null", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + process.handle?.finish(undefined); + + await expect(session.wait()).resolves.toEqual({ exitCode: null }); + }); + + it("kills the child and closes the session", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.kill(); + expect(process.handle?.killed).toBe(1); + + await session.close(); + expect(process.handle?.disconnected).toBe(1); + }); + + it("passes the working directory to the pseudo-terminal", async () => { + const process = createFakeProcess(); + + await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE, { cwd: "/workspace" }); + + expect(process.createOptions?.cwd).toBe("/workspace"); + }); +}); + +describe("openDaytonaLoginPtySession — ancestor symlink", () => { + it("starts no pseudo-terminal when the home filesystem check fails closed", async () => { + // A symlinked login-root ancestor makes the descriptor-relative inspection fail + // closed. The session opener must reject before it opens the terminal, so no + // pseudo-terminal starts and the login command never runs. + const process = createFakeProcess(); + let created = false; + const fs: DaytonaLoginHomeFs = { + async loginUserId(): Promise { + return LOGIN_UID; + }, + async inspect(): Promise { + throw new Error("LOGIN_PTY_HOME_REJECTED"); + }, + async createDirectory(): Promise { + created = true; + }, + }; + + await expect(openDaytonaLoginPtySession(process, fs, CODEX)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + // The provider never created the directory and never opened the terminal. + expect(created).toBe(false); + expect(process.createCount).toBe(0); + }); +}); + +describe("createDaytonaLoginHomeFs — login-profile preamble", () => { + // A capturing exec surface. It records each command and returns a fixed result, + // so a test reads the exact command string the helper runs. + function createCapturingExec(result: DaytonaExecResult): DaytonaSandboxExec & { + commands: string[]; + } { + const commands: string[] = []; + return { + commands, + async executeCommand(command: string): Promise { + commands.push(command); + // `id -u` resolves the login user id. Return a fixed uid so the caller + // continues to the node helper command under test. + if (command.startsWith("id -u")) { + return { exitCode: 0, result: "1000" }; + } + return result; + }, + }; + } + + // The command sources /etc/profile before it runs node. The Daytona image may + // expose node only through a login profile, so node must run after the profile + // source. This assertion proves the helper runtime and the login PTY capability + // stay consistent. + function expectProfileBeforeNode(command: string | undefined): void { + expect(command).toBeDefined(); + const profileIndex = command!.indexOf("/etc/profile"); + const nodeIndex = command!.indexOf("node -e"); + expect(profileIndex).toBeGreaterThanOrEqual(0); + expect(nodeIndex).toBeGreaterThanOrEqual(0); + expect(profileIndex).toBeLessThan(nodeIndex); + } + + it("sources the login profiles before it runs the node inspect helper", async () => { + const exec = createCapturingExec({ exitCode: 0, result: "ABSENT" }); + const fs = createDaytonaLoginHomeFs(exec); + await fs.inspect(HOME); + expectProfileBeforeNode(exec.commands.find((command) => command.includes("node -e"))); + }); + + it("sources the login profiles before it runs the node create helper", async () => { + const exec = createCapturingExec({ exitCode: 0, result: "" }); + const fs = createDaytonaLoginHomeFs(exec); + await fs.createDirectory(HOME, "700"); + expectProfileBeforeNode(exec.commands.find((command) => command.includes("node -e"))); + }); +}); + +// The real login-home filesystem runs a node helper on the sandbox. The helper +// walks the path with `/proc/self/fd`, which is Linux only. A non-Linux host skips +// these tests. The fake exec runs the helper locally, so the descriptor-relative +// walk is tested against a real filesystem. The login sandbox is Linux. +const describeLinux = process.platform === "linux" ? describe : describe.skip; + +describeLinux("createDaytonaLoginHomeFs — descriptor-relative walk", () => { + let root: string; + + beforeAll(() => { + root = mkdtempSync(path.join(tmpdir(), "daytona-login-home-")); + }); + afterAll(() => { + rmSync(root, { recursive: true, force: true }); + }); + + // A local exec surface. It runs the composed command with `/bin/sh -c`, so the + // real node helper runs against the local filesystem. + function createLocalExec(): DaytonaSandboxExec { + return { + async executeCommand(command: string): Promise { + const result = spawnSync("/bin/sh", ["-c", command], { encoding: "utf8" }); + return { exitCode: result.status ?? undefined, result: result.stdout ?? "" }; + }, + }; + } + + const UUID = "11111111-2222-4333-8444-555555555555"; + + it("reports a not-yet-created target as absent", async () => { + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const home = path.join(root, "absent-root", UUID); + const inspection = await fs.inspect(home); + expect(inspection.exists).toBe(false); + }); + + it("creates the login root and the session home relative to a trusted descriptor", async () => { + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const home = path.join(root, "clean-root", UUID); + await fs.createDirectory(home, "700"); + const inspection = await fs.inspect(home); + expect(inspection.exists).toBe(true); + expect(inspection.isSymlink).toBe(false); + expect(inspection.isDirectory).toBe(true); + expect(inspection.mode).toBe("700"); + expect(inspection.ownerUid).toBe(process.getuid ? process.getuid() : inspection.ownerUid); + }); + + it("rejects a pre-existing session home target", async () => { + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const home = path.join(root, "exists-root", UUID); + await fs.createDirectory(home, "700"); + // A second create of the same target fails, because the target exists. + await expect(fs.createDirectory(home, "700")).rejects.toThrow("LOGIN_PTY_HOME_REJECTED"); + }); + + it("fails the inspection closed when the login root is a symlink", async () => { + // Pre-place a symlink at the login-root ancestor. The descriptor-relative walk + // opens the root with a no-follow open, so the inspection fails closed. + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const attacker = path.join(root, "inspect-attacker"); + mkdirSync(attacker, { mode: 0o700 }); + const rootLink = path.join(root, "inspect-symlink-root"); + symlinkSync(attacker, rootLink); + const home = path.join(rootLink, UUID); + await expect(fs.inspect(home)).rejects.toThrow("LOGIN_PTY_HOME_REJECTED"); + }); + + it("fails the creation closed when the login root is a pre-placed symlink", async () => { + // Pre-place a symlink at the login-root ancestor that points at an attacker + // tree. The creation opens the root with a no-follow open after the tolerant + // `mkdir`, so it fails closed and creates no directory in the attacker tree. + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const attacker = path.join(root, "create-attacker"); + mkdirSync(attacker, { mode: 0o700 }); + const rootLink = path.join(root, "create-symlink-root"); + symlinkSync(attacker, rootLink); + const home = path.join(rootLink, UUID); + await expect(fs.createDirectory(home, "700")).rejects.toThrow("LOGIN_PTY_HOME_REJECTED"); + // The creation placed no session home inside the attacker tree. + expect(existsSync(path.join(attacker, UUID))).toBe(false); + }); +}); + +describe("createDaytonaLoginPtySessionOpener", () => { + it("opens a session for the descriptor on each call", async () => { + const process = createFakeProcess(); + const opener = createDaytonaLoginPtySessionOpener(process, createFakeHomeFs(), { + cwd: "/workspace", + }); + + const session = await opener(CLAUDE); + + expect(process.createOptions?.cwd).toBe("/workspace"); + expect(process.handle?.inputs[0]).toBe("exec claude setup-token" + ENTER); + expect(typeof session.onData).toBe("function"); + expect(typeof session.write).toBe("function"); + expect(typeof session.wait).toBe("function"); + expect(typeof session.kill).toBe("function"); + expect(typeof session.close).toBe("function"); + }); +}); diff --git a/packages/plugins/sandbox-providers/daytona/src/login-pty.ts b/packages/plugins/sandbox-providers/daytona/src/login-pty.ts new file mode 100644 index 00000000000..803cafc729d --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/login-pty.ts @@ -0,0 +1,586 @@ +// The Daytona pseudo-terminal (PTY) session for the shared login flow. The login +// command needs a real pseudo-terminal: pipe stdio emits no login prompt. The +// Daytona SDK opens a PTY through `process.createPty`. This module binds that PTY +// to a small session that the login PTY transport consumes, so the login runner +// runs the command on a real terminal, streams the terminal output, and delivers +// the delayed browser code plus the Enter byte. +// +// Command contract: the host resolves a closed login command key and a validated +// session home, and it carries them in a launch descriptor. The descriptor holds +// no command string. This module maps the key to a compile-time command, so a +// caller cannot select or override the command. For the Codex key the module +// composes `exec env CODEX_HOME= `. For the +// Claude key it composes `exec ` with no CODEX_HOME. The +// module encodes the dynamic home with a POSIX shell-argument encoder, so the +// home cannot add a second shell token or a second command. +// +// Session home: the module revalidates the descriptor and the home shape, then +// creates the exact UUID directory as a NEW directory with mode 0700, owned by the +// login user. The inspection and the creation open the filesystem root, then walk +// each path component with a no-follow, directory-only open, so a symlink at any +// ancestor (the login root or a directory above it) fails closed. A single +// composite `stat` or `mkdir` follows a symlink at an ancestor; the per-component +// walk does not. The creation creates the login root if the root is absent and the +// UUID directory as a NEW directory, both relative to an open trusted descriptor. +// The module rejects a pre-existing target, a symlink, a non-directory, a wrong +// owner, and a wrong mode. It uses no recursive delete. It re-checks the directory +// with a no-symlink walk before it opens the terminal and again before it sends the +// launch input, so a symlink swapped in after creation fails before the launch. +// +// Dependency boundary: this provider plugin ships standalone (the workspace +// excludes `packages/plugins/sandbox-providers/**`). So the module imports no +// workspace package. It declares the small session shape locally through +// {@link LoginPtySession}. The shape matches the `LoginPtySession` interface in +// `@paperclipai/adapter-utils`, so the transport factory `createLoginPtyTransport` +// accepts the session opener from this module with no adapter. The runner drives +// the transport; the transport drives this session. +// +// SDK boundary: the module holds no Daytona SDK type import. It declares the small +// subset of the SDK PTY and filesystem surfaces that it uses through +// {@link DaytonaPtyProcess}, {@link DaytonaPtyHandle}, and {@link DaytonaSandboxExec}. +// The SDK `Process` and `PtyHandle` satisfy those subsets, so a caller passes +// `sandbox.process` with no adapter. The narrow surface keeps the module +// unit-testable with a fake PTY and a fake filesystem. +// +// Security (secret handling): the login runner delivers the browser code and the +// Enter byte through {@link DaytonaPtyHandle.sendInput}. The SDK sends the input +// over the PTY socket, so the code never rides a command line and never reaches a +// process argument list. The session forwards the raw terminal bytes with no ANSI +// or OSC 8 handling; the login parser owns that handling. + +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { sendPtyInputInChunks } from "./pty-chunked-input.js"; + +/** + * The closed set of login command identities. The worker maps each key to a + * compile-time command. A value outside this set fails closed before the module + * touches the filesystem. + */ +export type LoginCommandKey = "claude" | "codex"; + +/** + * The host-resolved launch descriptor. It carries the closed command key and the + * server-controlled session home. It holds no command string. + */ +export interface LoginPtyLaunchDescriptor { + /** The host-resolved fixed command identity. */ + loginCommandKey: LoginCommandKey; + /** + * The server-controlled session home. The shape is exact: + * `/tmp/paperclip-adapter-login/`. + */ + sessionHome: string; +} + +/** + * The compile-time command for each login command key. The Daytona plugin ships + * standalone, so it holds its own copy of the fixed commands. A future change to + * a command lands here and in the adapter constant together. + */ +const LOGIN_COMMAND_BY_KEY: Readonly> = { + claude: "claude setup-token", + codex: "codex login --device-auth", +}; + +/** The octal mode string for a new session home directory. */ +const LOGIN_HOME_MODE = "700"; + +/** The fixed root for a login session home. */ +const LOGIN_SESSION_HOME_ROOT = "/tmp/paperclip-adapter-login"; + +/** + * The exact absolute path shape for a login session home. The path is the fixed + * root, one slash, and one UUID. The anchors reject a relative path, a traversal, + * whitespace, a control character, and a shell metacharacter, because none of + * those match the UUID or the fixed root. + */ +const SESSION_HOME_PATTERN = + /^\/tmp\/paperclip-adapter-login\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +/** The fixed non-secret error a rejected launch descriptor or session home returns. */ +export const LOGIN_PTY_DESCRIPTOR_REJECTED = "LOGIN_PTY_DESCRIPTOR_REJECTED"; +/** The fixed non-secret error a rejected session home directory returns. */ +export const LOGIN_PTY_HOME_REJECTED = "LOGIN_PTY_HOME_REJECTED"; + +/** + * Encodes one value as a single POSIX shell argument. It wraps the value in + * single quotes and escapes each embedded single quote. The shell treats every + * other character inside single quotes as a literal, so an encoded value cannot + * add a second shell token or a second command. + */ +export function encodePosixShellArg(value: string): string { + return `'${value.split("'").join("'\\''")}'`; +} + +/** Reports whether a value is a member of the closed login command key set. */ +export function isLoginCommandKey(value: unknown): value is LoginCommandKey { + return value === "claude" || value === "codex"; +} + +/** + * Composes the launch line for the descriptor. For the Codex key it prefixes one + * safely encoded `CODEX_HOME` assignment with `env`. For the Claude key it adds no + * `CODEX_HOME`. It replaces the interactive shell with the command through `exec`, + * so the pseudo-terminal runs the command directly and its exit code becomes the + * PTY exit code. + */ +export function composeLaunchLine(descriptor: LoginPtyLaunchDescriptor): string { + const command = LOGIN_COMMAND_BY_KEY[descriptor.loginCommandKey]; + if (descriptor.loginCommandKey === "codex") { + const encodedHome = encodePosixShellArg(descriptor.sessionHome); + return `exec env CODEX_HOME=${encodedHome} ${command}`; + } + return `exec ${command}`; +} + +/** + * A live pseudo-terminal session for one login command. The session allocates a + * real pseudo-terminal, streams the raw terminal output, accepts delayed input, + * and stops the child. The shape matches the `LoginPtySession` interface in + * `@paperclipai/adapter-utils`, so the transport factory there accepts a session + * opener that returns this session. + */ +export interface LoginPtySession { + /** Registers the one output listener. The session streams each raw chunk in order. */ + onData(listener: (chunk: string) => void): void; + /** Writes raw input bytes to the pseudo-terminal. */ + write(data: string): void; + /** Resolves with the child exit code when the command ends. */ + wait(): Promise<{ exitCode: number | null }>; + /** Stops the child process. Safe to call more than one time. */ + kill(): void; + /** Releases the session resources. Safe to call more than one time. */ + close(): Promise; +} + +/** Opens a {@link LoginPtySession} for `descriptor`. The transport calls it one time. */ +export type LoginPtySessionOpener = ( + descriptor: LoginPtyLaunchDescriptor, +) => Promise; + +/** + * The subset of the Daytona SDK `PtyHandle` that the session uses. The SDK + * `PtyHandle` satisfies this interface, so a caller passes the real handle with + * no adapter. + */ +export interface DaytonaPtyHandle { + /** Resolves when the PTY socket connection is ready for input and output. */ + waitForConnection(): Promise; + /** Sends raw input bytes to the pseudo-terminal. */ + sendInput(data: string | Uint8Array): Promise; + /** Resolves with the exit result when the pseudo-terminal process ends. */ + wait(): Promise<{ exitCode?: number; error?: string }>; + /** Kills the pseudo-terminal process. */ + kill(): Promise; + /** Closes the PTY socket and releases the resources. */ + disconnect(): Promise; +} + +/** + * The options the session passes to `createPty`. The session sets a fixed + * terminal size and one output callback. It sets the working directory when the + * caller provides one. + */ +export interface DaytonaPtyCreateOptions { + id: string; + cwd?: string; + cols: number; + rows: number; + onData: (data: Uint8Array) => void | Promise; +} + +/** + * The subset of the Daytona SDK `Process` that the session uses. The SDK + * `Process` satisfies this interface, so a caller passes `sandbox.process`. + */ +export interface DaytonaPtyProcess { + createPty(options: DaytonaPtyCreateOptions): Promise; +} + +/** The result of one command run on the sandbox. */ +export interface DaytonaExecResult { + exitCode?: number; + result?: string; +} + +/** + * The subset of the Daytona SDK `Process` that the filesystem helper uses. The + * SDK `Process` satisfies this interface through `executeCommand`, so a caller + * passes `sandbox.process`. + */ +export interface DaytonaSandboxExec { + executeCommand( + command: string, + cwd?: string, + env?: Record, + timeout?: number, + ): Promise; +} + +/** One no-follow inspection of a filesystem path. */ +export interface LoginHomeInspection { + /** True when the path exists (as any type). */ + exists: boolean; + /** True when the path itself is a symbolic link. */ + isSymlink: boolean; + /** True when the path itself is a directory. */ + isDirectory: boolean; + /** The octal permission string, for example `700`. Empty when the path is absent. */ + mode: string; + /** The owner user id, or null when the path is absent. */ + ownerUid: number | null; +} + +/** + * The narrow filesystem surface the session home operations need. The production + * surface runs commands on the sandbox; a unit test injects a fake. The inspection + * and the creation walk each path component with a no-follow open, so a symlink at + * any ancestor fails closed and a symlink at the target reports the link. + */ +export interface DaytonaLoginHomeFs { + /** Resolves the user id of the login user that runs the pseudo-terminal. */ + loginUserId(): Promise; + /** + * Inspects `path` without following a symlink at the target or at any ancestor. + * It rejects (throws) when an ancestor is a symlink or a non-directory. + */ + inspect(path: string): Promise; + /** + * Creates `path` as a NEW directory with the octal `mode`. It creates the login + * root ancestor if the root is absent. It fails when the target path exists or + * when an ancestor is a symlink. It follows no composite pathname. + */ + createDirectory(path: string, mode: string): Promise; +} + +/** The options for the Daytona login PTY session. */ +export interface DaytonaLoginPtyOptions { + /** The working directory for the login PTY. Defaults to the sandbox default. */ + cwd?: string; +} + +// The terminal size for the login PTY. The login UI prints a short prompt, so a +// standard terminal size is enough. A fixed size keeps the output stable. +const LOGIN_PTY_COLS = 120; +const LOGIN_PTY_ROWS = 30; + +/** + * The input terminator that replaces the interactive shell with the login + * command. The `exec` builtin replaces the shell process image with the command, + * so the PTY runs the command directly. When the command ends, the PTY ends with + * the command exit code, and the delayed input reaches the command, not a shell. + * The terminal reads the carriage return as the Enter key. + */ +const PTY_COMMAND_TERMINATOR = "\r"; + +/** Normalizes an octal permission string to its numeric value for comparison. */ +function octalMode(value: string): number { + return Number.parseInt(value, 8); +} + +/** + * Revalidates the launch descriptor. It fails closed when the command key is + * outside the closed set or the session home shape is wrong. A unit test that + * bypasses the host boundary hits this check, so the provider never trusts an + * unvalidated descriptor. + */ +function assertDescriptor(descriptor: LoginPtyLaunchDescriptor): void { + if (!isLoginCommandKey(descriptor.loginCommandKey)) { + throw new Error(LOGIN_PTY_DESCRIPTOR_REJECTED); + } + if (!SESSION_HOME_PATTERN.test(descriptor.sessionHome)) { + throw new Error(LOGIN_PTY_DESCRIPTOR_REJECTED); + } +} + +/** + * Creates and validates the exact session home directory. It rejects a + * pre-existing target (including a symlink), creates the directory as a NEW + * directory with mode 0700, then verifies the directory type, the no-symlink + * state, the mode, and the login-user ownership. It uses no recursive delete. + */ +async function prepareSessionHome(fs: DaytonaLoginHomeFs, sessionHome: string): Promise { + const loginUid = await fs.loginUserId(); + + // Reject a pre-existing target. A no-follow inspection reports a symlink, a + // file, or a directory as present, so any pre-existing target fails here. + const before = await fs.inspect(sessionHome); + if (before.exists) { + throw new Error(LOGIN_PTY_HOME_REJECTED); + } + + // Create the exact directory as a NEW directory with mode 0700. The create + // fails when the path exists, so the create never follows a symlink. + await fs.createDirectory(sessionHome, LOGIN_HOME_MODE); + + // Verify the created directory. Reject a symlink, a non-directory, a wrong + // owner, and a wrong mode. Do not delete on a rejection; the sandbox is + // ephemeral and the provider uses no recursive delete. + const after = await fs.inspect(sessionHome); + if ( + !after.exists || + after.isSymlink || + !after.isDirectory || + after.ownerUid !== loginUid || + octalMode(after.mode) !== octalMode(LOGIN_HOME_MODE) + ) { + throw new Error(LOGIN_PTY_HOME_REJECTED); + } +} + +/** + * Re-checks the directory with a no-symlink check. It rejects a symlink and a + * non-directory, so a symlink swapped in after creation fails before the launch. + */ +async function assertHomeStillSafe(fs: DaytonaLoginHomeFs, sessionHome: string): Promise { + const now = await fs.inspect(sessionHome); + if (!now.exists || now.isSymlink || !now.isDirectory) { + throw new Error(LOGIN_PTY_HOME_REJECTED); + } +} + +/** + * Opens a Daytona PTY session for `descriptor` and returns it as a + * {@link LoginPtySession}. The function revalidates the descriptor and the home + * shape, creates and validates the session home, opens a real pseudo-terminal, + * re-checks the directory before the launch, composes the safe launch line, and + * sends it. The session streams the raw terminal output, delivers delayed input, + * and stops the child. + * + * The function decodes the terminal bytes as a UTF-8 stream, so a multibyte + * character that splits across two output chunks stays whole. It buffers the + * output until the transport registers the listener, so no early chunk is lost. + */ +export async function openDaytonaLoginPtySession( + process: DaytonaPtyProcess, + fs: DaytonaLoginHomeFs, + descriptor: LoginPtyLaunchDescriptor, + options?: DaytonaLoginPtyOptions, +): Promise { + assertDescriptor(descriptor); + // Create and validate the session home before the terminal opens. + await prepareSessionHome(fs, descriptor.sessionHome); + // Re-check the directory with a no-symlink check before `createPty`. + await assertHomeStillSafe(fs, descriptor.sessionHome); + + const decoder = new TextDecoder("utf-8"); + let listener: ((chunk: string) => void) | null = null; + let buffered = ""; + + const handle = await process.createPty({ + id: `paperclip-login-pty-${randomUUID()}`, + ...(options?.cwd ? { cwd: options.cwd } : {}), + cols: LOGIN_PTY_COLS, + rows: LOGIN_PTY_ROWS, + onData: (data: Uint8Array): void => { + // Decode the terminal bytes as a stream, so a split multibyte character + // stays whole across two chunks. Forward the raw text; the parser owns the + // ANSI and OSC 8 handling. + const text = decoder.decode(data, { stream: true }); + if (text.length === 0) return; + if (listener) listener(text); + else buffered += text; + }, + }); + + await handle.waitForConnection(); + // Re-check the directory with a no-symlink check immediately before the launch + // input, so a symlink swapped in after creation fails before `sendInput`. + await assertHomeStillSafe(fs, descriptor.sessionHome); + // Replace the interactive shell with the login command, so the pseudo-terminal + // runs the command directly. The runner then writes the delayed browser code + // to the command, not to a shell. + await handle.sendInput(composeLaunchLine(descriptor) + PTY_COMMAND_TERMINATOR); + + // The tail of the write chain. The chunker awaits each chunk, so one write can + // suspend between its chunks. The chain runs each write after the previous + // write ends, so the chunks of two writes never interleave on the wire. + let writeChain: Promise = Promise.resolve(); + + return { + onData(next: (chunk: string) => void): void { + listener = next; + if (buffered.length > 0) { + const pending = buffered; + buffered = ""; + next(pending); + } + }, + write(data: string): void { + // Send the input as byte-bounded chunks under the provider message cap. The + // login input is short today, so the latent defect never fires, but the same + // unbounded write goes through the one chunker. The chain runs this write + // after the previous write ends, so two writes keep their order and never + // interleave their chunks. A write error must not throw into the runner, so + // the runner's fixed status stays the single result path. + writeChain = writeChain.then(() => + sendPtyInputInChunks((chunk) => handle.sendInput(chunk), data), + ); + }, + async wait(): Promise<{ exitCode: number | null }> { + const result = await handle.wait(); + return { exitCode: typeof result.exitCode === "number" ? result.exitCode : null }; + }, + kill(): void { + void handle.kill().catch(() => undefined); + }, + async close(): Promise { + await handle.disconnect().catch(() => undefined); + }, + }; +} + +/** + * The inspection helper source. The provider reads it from its own script file and + * runs it on the sandbox as ` && node -e