diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.ts index 6f5ae7dff6d..217e4f582d1 100644 --- a/packages/adapter-utils/src/acpx-engine/execute.test.ts +++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts @@ -5763,8 +5763,8 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => // Wrap a started real broker in a paperclip bridge handle. The handle exposes // the same run-disposition surface the sandbox bridge exposes, so the seam runs // against the real latch and the real orderly-completion mark. - function bridgeOverBroker(fake: ReturnType) { - const broker = createDuplexBridgeBroker({ + async function bridgeOverBroker(fake: ReturnType) { + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -5891,7 +5891,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("fails a completed run when the duplex channel was lost before the completion", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake); + const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); // Latch the loss before the ACP terminal resolves. const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); @@ -5912,7 +5912,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("keeps a completed run a success when the channel stays live, and a later teardown loss is benign", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, settleRunDisposition } = bridgeOverBroker(fake); + const { broker, handle, settleRunDisposition } = await bridgeOverBroker(fake); // No loss before the completion. const runtime = runtimeWithControlledResult(); @@ -5932,7 +5932,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("does not let a later completion or activity clear the loss latch", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle } = bridgeOverBroker(fake); + const { broker, handle } = await bridgeOverBroker(fake); // Latch the loss before the ACP terminal resolves. const runtime = runtimeWithControlledResult(() => fake.emitExit({ exitCode: 1 })); @@ -5951,7 +5951,7 @@ describe("ACPX engine sandbox duplex run-disposition seam (fail-closed)", () => it("marks an orderly completion on a failed terminal so the teardown loss emits no false loss", async () => { const sandbox = await setupRemoteSandbox(); const fake = createFakeDuplexChannel(); - const { broker, handle, markOrderlyCompletion } = bridgeOverBroker(fake); + const { broker, handle, markOrderlyCompletion } = await bridgeOverBroker(fake); // The turn fails, and no channel loss ordered before the finalization. const runtime = runtimeWithFailedResult(); diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index 4c879dbf4da..1121d8365d5 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -40,8 +40,13 @@ export interface CommandManagedDuplexChannel { write(data: string): void; /** Registers the one data listener. The channel streams each raw chunk in order. */ onData(listener: (chunk: string) => void): void; - /** Registers the one exit listener. The channel calls it one time with the exit. */ - onExit(listener: (exit: { exitCode: number | null }) => void): void; + /** + * Registers the one exit listener. The channel calls it one time with the exit. + * A numeric `exitCode` is a real process exit. `transportClosed` is true when the + * provider transport closed with no exit data, so a reader can tell a real + * process exit from a reason-less transport close. + */ + onExit(listener: (exit: { exitCode: number | null; transportClosed?: boolean }) => void): void; /** Stops the child process. Safe to call more than one time. */ stop(): void; /** Closes the channel and releases the route. Safe to call more than one time. */ diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts new file mode 100644 index 00000000000..790100509e8 --- /dev/null +++ b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + DUPLEX_AGGREGATE_TOKEN_OWNERS, + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + DuplexAggregateByteLedger, + MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX, + assertDuplexAggregateCeilingBytes, + resolveDuplexAggregateCeilingBytes, + type DuplexAggregateByteLedgerTelemetry, +} from "./duplex-aggregate-byte-ledger.js"; + +// A telemetry surface that counts each fixed record, so a test asserts the exact +// gauge value and the exact counter counts. +function createCountingTelemetry(): DuplexAggregateByteLedgerTelemetry & { + gauge: number; + rejections: number; + underflows: number; +} { + const state = { gauge: 0, rejections: 0, underflows: 0 }; + return { + ...state, + setBytesInUse(bytes: number) { + this.gauge = bytes; + }, + recordReservationRejection() { + this.rejections += 1; + }, + recordAccountingUnderflow() { + this.underflows += 1; + }, + }; +} + +describe("DuplexAggregateByteLedger", () => { + it("reserves under the ceiling and tracks the gauge and the live-token count", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + const a = ledger.reserve("buffered_chunk", 400); + const b = ledger.reserve("buffered_chunk", 500); + + expect(a).not.toBeNull(); + expect(b).not.toBeNull(); + expect(ledger.bytesInUse).toBe(900); + expect(ledger.liveTokenCount).toBe(2); + expect(telemetry.gauge).toBe(900); + expect(a?.bytes).toBe(400); + expect(a?.owner).toBe("buffered_chunk"); + expect(a?.state).toBe("held"); + }); + + it("rejects a one-byte-over reservation, retains nothing, and increments the rejection counter", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + ledger.reserve("buffered_chunk", 1000); + const over = ledger.reserve("buffered_chunk", 1); + + expect(over).toBeNull(); + expect(ledger.bytesInUse).toBe(1000); + expect(ledger.liveTokenCount).toBe(1); + expect(telemetry.rejections).toBe(1); + // The public rejection marker is the fixed string the caller reports. + expect(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED).toBe("DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED"); + }); + + it("transfers ownership without a decrement or a re-reserve, so no admission gap opens", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 100, telemetry }); + + // Reserve the whole ceiling as a pre-bind event, then transfer to buffered. + const token = ledger.reserve("pre_bind_event", 100); + expect(token).not.toBeNull(); + // A second reservation cannot fit while the token stays held. + expect(ledger.reserve("buffered_chunk", 1)).toBeNull(); + + ledger.transfer(token!, "buffered_chunk"); + + // The transfer changed only the owner. The gauge never moved, so no window + // opened where the bytes were released and re-admitted. + expect(ledger.bytesInUse).toBe(100); + expect(ledger.liveTokenCount).toBe(1); + expect(token?.owner).toBe("buffered_chunk"); + expect(telemetry.gauge).toBe(100); + // No reservation fit during the transfer, so no rejection or underflow fired + // beyond the one rejection above. + expect(telemetry.rejections).toBe(1); + expect(telemetry.underflows).toBe(0); + }); + + it("releases a token one time and decrements the gauge", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + const token = ledger.reserve("response_body", 250); + expect(token).not.toBeNull(); + ledger.release(token!); + + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(token?.state).toBe("released"); + expect(telemetry.gauge).toBe(0); + expect(telemetry.underflows).toBe(0); + }); + + it("records the underflow counter on a second release and does not clamp the gauge again", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + + const kept = ledger.reserve("request_frame", 300); + const doubled = ledger.reserve("request_frame", 200); + expect(kept).not.toBeNull(); + expect(doubled).not.toBeNull(); + ledger.release(doubled!); + expect(ledger.bytesInUse).toBe(300); + + // A forced second release of the same token is an accounting defect. + ledger.release(doubled!); + + expect(telemetry.underflows).toBe(1); + // The gauge did not drop a second time, so the defect stays visible. + expect(ledger.bytesInUse).toBe(300); + expect(ledger.liveTokenCount).toBe(1); + }); + + it("records the underflow counter on a transfer of a released token", () => { + const telemetry = createCountingTelemetry(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000, telemetry }); + const token = ledger.reserve("pre_bind_event", 100); + ledger.release(token!); + + ledger.transfer(token!, "buffered_chunk"); + + expect(telemetry.underflows).toBe(1); + expect(ledger.bytesInUse).toBe(0); + }); + + it("throws on a non-integer or negative reservation, so a programming error fails loud", () => { + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 }); + expect(() => ledger.reserve("buffered_chunk", -1)).toThrow(); + expect(() => ledger.reserve("buffered_chunk", 1.5)).toThrow(); + expect(() => ledger.reserve("buffered_chunk", Number.NaN)).toThrow(); + }); + + it("admits a zero-byte reservation as a valid held token", () => { + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1000 }); + const token = ledger.reserve("seen_request_id", 0); + expect(token).not.toBeNull(); + expect(token?.bytes).toBe(0); + expect(ledger.liveTokenCount).toBe(1); + }); + + it("exposes the closed owner-label set", () => { + expect([...DUPLEX_AGGREGATE_TOKEN_OWNERS]).toEqual([ + "pre_bind_event", + "buffered_chunk", + "terminal_buffered", + "request_frame", + "request_payload", + "response_body", + "seen_request_id", + "decoder_buffer", + "readiness_buffer", + "readiness_replay", + "pending_write", + "stdin_write", + ]); + }); +}); + +describe("aggregate ceiling configuration", () => { + it("uses the default when the override is absent", () => { + expect(resolveDuplexAggregateCeilingBytes()).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + expect(resolveDuplexAggregateCeilingBytes(null)).toBe(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES); + expect(DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES).toBe(256 * 1024 * 1024); + }); + + it("accepts a valid positive override under the process allocation minus the reserve", () => { + expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024)).toBe(64 * 1024 * 1024); + const maxCeiling = + MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; + expect(resolveDuplexAggregateCeilingBytes(maxCeiling)).toBe(maxCeiling); + }); + + it("rejects an invalid override to the safe default and reports each rejection", () => { + const overMaximum = MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX; + const invalidOverrides = [0, -1, 1.5, Number.POSITIVE_INFINITY, Number.NaN, overMaximum]; + for (const invalid of invalidOverrides) { + const rejected: number[] = []; + // An invalid present override returns the safe default. It never throws, so a + // single bad environment value never fails host startup. + expect(resolveDuplexAggregateCeilingBytes(invalid, (value) => rejected.push(value))).toBe( + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + ); + // The resolver reports the rejection one time with the rejected numeric value. + expect(rejected).toEqual([invalid]); + } + }); + + it("does not report a rejection for a valid or a missing override", () => { + const rejected: number[] = []; + const reporter = (value: number): void => { + rejected.push(value); + }; + expect(resolveDuplexAggregateCeilingBytes(64 * 1024 * 1024, reporter)).toBe(64 * 1024 * 1024); + expect(resolveDuplexAggregateCeilingBytes(undefined, reporter)).toBe( + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + ); + expect(resolveDuplexAggregateCeilingBytes(null, reporter)).toBe( + DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES, + ); + expect(rejected).toEqual([]); + }); + + it("still throws for a direct ceiling assertion on an unsafe integer", () => { + // The ledger constructor asserts its ceiling and must fail loud on a + // programming error, so `assertDuplexAggregateCeilingBytes` keeps throwing. + expect(() => assertDuplexAggregateCeilingBytes(Number.MAX_SAFE_INTEGER + 2)).toThrow(); + }); +}); diff --git a/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts new file mode 100644 index 00000000000..e7f7aec3c16 --- /dev/null +++ b/packages/adapter-utils/src/duplex-aggregate-byte-ledger.ts @@ -0,0 +1,400 @@ +/** + * The process-owned aggregate byte ledger for the sandbox duplex channel. + * + * The route-count controller bounds how many concurrent duplex routes stay live. + * It does not bound the aggregate bytes those routes retain. At the maximum route + * count the per-route byte bounds multiply to many gigabytes of retained bytes. + * This ledger closes that gap. One ledger per host server process owns a single + * byte ceiling. Every host-side retention site reserves its exact retained bytes + * against this ledger before it allocates, so the aggregate retained bytes across + * all live routes never pass the ceiling. + * + * The ledger is fail-closed. A reservation that would pass the ceiling returns no + * token. The caller retains nothing and reports the fixed rejection marker + * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. + * + * Ownership is transfer-only. A token holds an immutable byte amount, an owner + * label from a closed enum, and a one-way state: `held` then `released`. The + * ledger moves a token from one representation to the next with + * {@link DuplexAggregateByteLedger.transfer}, which changes only the owner label. + * It never decrements and re-reserves, so no admission gap opens between two + * representations of the same retained bytes. + * + * The ledger holds every live token in one owner registry. Registry removal and + * the release of a token run in one synchronous critical section, before any + * `await` or external callback. A second release of the same token is an + * accounting defect. The ledger records the fixed underflow counter and does not + * clamp the gauge down a second time, so a real defect stays visible. + * + * The ledger carries no payload data and no arbitrary labels. It exposes only the + * byte gauge, the live-token count, and the closed owner enum. + */ + +import { + DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, + DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, + DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, +} from "./duplex-telemetry.js"; + +/** + * The public rejection marker. A retention site that cannot reserve its bytes + * reports this fixed string. It carries no payload, no route, and no raw value. + */ +export const DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED = "DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED"; + +/** + * The default aggregate ceiling, in bytes: 256 MiB. This is one eighth of the + * documented minimum host process memory allocation for the sandbox duplex + * transport ({@link MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX}, 2 GiB). The + * remaining seven eighths cover the Node runtime, ordinary server work, parser + * and transient headroom, and allocator or garbage-collector variance. + */ +export const DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES = 256 * 1024 * 1024; + +/** + * The documented minimum host process memory allocation to enable the sandbox + * duplex transport, in bytes: 2 GiB. An operator override of the aggregate + * ceiling must not pass this value minus the documented reserve. + */ +export const MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX = 2 * 1024 * 1024 * 1024; + +/** + * The closed set of owner labels a reservation token can carry. Each label names + * one host-side representation that retains bytes. The ledger accepts no label + * outside this set, so no arbitrary label ever reaches a token. + * + * - `pre_bind_event`: a held pre-bind duplex event, before route bind. + * - `buffered_chunk`: a buffered data chunk retained for a late listener. + * - `terminal_buffered`: a buffered chunk moved to the terminal registry at + * route terminalization, held for a late listener drain. + * - `request_frame`: a bounded raw request frame the broker retains at dispatch. + * - `request_payload`: a normalized request payload the broker retains at dispatch. + * - `response_body`: a response-body chunk or replacement buffer the reader retains. + * - `seen_request_id`: one entry of the broker no-replay request-id set. + * - `decoder_buffer`: the raw partial-frame bytes the host frame decoder retains + * between chunks, plus the peak replacement buffer it allocates on concat. + * - `readiness_buffer`: the raw untrusted bytes the readiness gate retains before + * the READY frame completes, before the broker binds and the decoder takes over. + * - `readiness_replay`: the post-READY suffix bytes the readiness gate holds in the + * pending replay buffer, from the READY accept until the broker binds and the + * frame decoder charges its own retention. This is a separate retention from + * `readiness_buffer`: the gate drops the whole pre-READY buffer on READY, then + * charges only the retained suffix and each later pre-bind chunk under this owner. + * - `pending_write`: the raw host-to-worker write payload a pending duplex write + * RPC retains, from the enqueue seam until the RPC settles. + * - `stdin_write`: the serialized host-to-worker frame the child-stdin transport + * buffer retains, from the write until the stream flushes the chunk, the stream + * errors, the stream closes, or the worker exits. This is a separate retention + * from `pending_write`: the RPC holds the raw payload while the transport buffer + * holds the larger serialized frame, so the two tokens cover the peak of both. + */ +export const DUPLEX_AGGREGATE_TOKEN_OWNERS = [ + "pre_bind_event", + "buffered_chunk", + "terminal_buffered", + "request_frame", + "request_payload", + "response_body", + "seen_request_id", + "decoder_buffer", + "readiness_buffer", + "readiness_replay", + "pending_write", + "stdin_write", +] as const; + +/** One owner label from the closed {@link DUPLEX_AGGREGATE_TOKEN_OWNERS} set. */ +export type DuplexAggregateTokenOwner = (typeof DUPLEX_AGGREGATE_TOKEN_OWNERS)[number]; + +/** The one-way lifecycle state of a reservation token. */ +export type DuplexReservationTokenState = "held" | "released"; + +/** + * An opaque reservation token. The byte amount is immutable. The owner label + * moves through the closed enum by transfer only. The state moves one way, from + * `held` to `released`. A caller reads these fields but never mutates them; only + * the owning ledger changes the owner and the state. + */ +export interface ReservationToken { + /** The immutable exact byte amount this token reserves. */ + readonly bytes: number; + /** The current owner label. The ledger changes it only through `transfer`. */ + readonly owner: DuplexAggregateTokenOwner; + /** The one-way lifecycle state. The ledger sets it to `released` one time. */ + readonly state: DuplexReservationTokenState; +} + +/** + * The internal mutable token. The ledger owns it and returns it typed as the + * read-only {@link ReservationToken}. Only code in this module changes the owner + * or the state. + */ +class ReservationTokenImpl implements ReservationToken { + readonly bytes: number; + owner: DuplexAggregateTokenOwner; + state: DuplexReservationTokenState = "held"; + + constructor(bytes: number, owner: DuplexAggregateTokenOwner) { + this.bytes = bytes; + this.owner = owner; + } +} + +/** + * The fixed telemetry surface for the aggregate byte ledger. The host binds it to + * the closed duplex telemetry contract. Each record uses only fixed constant + * dimensions. No company, agent, run, route, URL, header, request id, token, byte + * payload, or raw provider value ever reaches a record. The default is a no-op. + */ +export interface DuplexAggregateByteLedgerTelemetry { + /** Set the aggregate-bytes-in-use gauge to the current value. */ + setBytesInUse(bytes: number): void; + /** Increment the reservation-rejection counter one time. */ + recordReservationRejection(): void; + /** Increment the accounting-underflow counter one time. */ + recordAccountingUnderflow(): void; +} + +/** A no-op telemetry surface. Every method does nothing, so the ledger stays inert. */ +export const NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY: DuplexAggregateByteLedgerTelemetry = { + setBytesInUse() {}, + recordReservationRejection() {}, + recordAccountingUnderflow() {}, +}; + +/** + * The low-level process metric sink the host binds. It carries the fixed metric + * name and the value only, so no dynamic dimension reaches a sink. The host maps + * the gauge and the two counters to the process metric pipeline and to a warn log + * for a defect. + */ +export interface DuplexAggregateByteLedgerMetricSink { + /** Set a named process gauge to a value. */ + setGauge(name: string, value: number): void; + /** Increment a named process counter one time. */ + incrementCounter(name: string): void; +} + +/** + * Build a {@link DuplexAggregateByteLedgerTelemetry} that maps each ledger event + * to the fixed metric name on the process metric sink. The gauge uses + * {@link DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE}. The rejection counter uses + * {@link DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL}. The defect + * counter uses {@link DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL}. + * Each call sits in an error swallow, so a sink failure never breaks the reserve + * or the release path. + */ +export function createDuplexAggregateByteLedgerTelemetry( + sink: DuplexAggregateByteLedgerMetricSink, +): DuplexAggregateByteLedgerTelemetry { + return { + setBytesInUse(bytes: number): void { + try { + sink.setGauge(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, bytes); + } catch { + // A telemetry failure never breaks the reserve or the release path. + } + }, + recordReservationRejection(): void { + try { + sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL); + } catch { + // A telemetry failure never breaks the reserve path. + } + }, + recordAccountingUnderflow(): void { + try { + sink.incrementCounter(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL); + } catch { + // A telemetry failure never breaks the release path. + } + }, + }; +} + +/** The options for {@link DuplexAggregateByteLedger}. */ +export interface DuplexAggregateByteLedgerOptions { + /** The aggregate ceiling, in bytes. It must be a positive safe integer. */ + ceilingBytes: number; + /** The fixed telemetry surface. The default is the no-op surface. */ + telemetry?: DuplexAggregateByteLedgerTelemetry; +} + +/** + * Assert an aggregate ceiling byte value. The value must be a positive safe + * integer that is no larger than the documented process allocation minus the + * documented reserve. The function throws on any other value, so an invalid + * ceiling fails startup instead of silently disabling the ledger. + * + * The reserve equals the documented process allocation minus the default + * ceiling ({@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}), so the maximum + * accepted ceiling is exactly the documented process allocation minus that + * reserve. + */ +export function assertDuplexAggregateCeilingBytes(value: number): void { + const maxCeiling = maxDuplexAggregateCeilingBytes(); + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error( + `The duplex aggregate byte ceiling must be a positive safe integer; got ${String(value)}.`, + ); + } + if (value > maxCeiling) { + throw new Error( + `The duplex aggregate byte ceiling ${value} must not pass the documented process allocation minus the reserve (${maxCeiling}).`, + ); + } +} + +/** + * The maximum accepted aggregate ceiling, in bytes: the documented process + * allocation minus the documented reserve. Both {@link assertDuplexAggregateCeilingBytes} + * and {@link isValidDuplexAggregateCeilingBytes} read this one bound. + */ +function maxDuplexAggregateCeilingBytes(): number { + return MIN_HOST_PROCESS_MEMORY_BYTES_FOR_DUPLEX - DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; +} + +/** + * Report whether `value` is a valid aggregate ceiling. A valid ceiling is a + * positive safe integer that is no larger than {@link maxDuplexAggregateCeilingBytes}. + * This is the non-throwing form of {@link assertDuplexAggregateCeilingBytes}. The + * override resolver uses it to reject an invalid override without a throw. + */ +export function isValidDuplexAggregateCeilingBytes(value: number): boolean { + return Number.isSafeInteger(value) && value >= 1 && value <= maxDuplexAggregateCeilingBytes(); +} + +/** + * A reporter the host binds to receive one invalid-override rejection signal. + * {@link resolveDuplexAggregateCeilingBytes} calls it one time when it rejects a + * present invalid override and falls back to the safe default. The signal carries + * only the rejected numeric value. It carries no route, company, run, or payload + * value. + */ +export type DuplexAggregateCeilingOverrideRejectionReporter = (rejectedValue: number) => void; + +/** + * Resolve an aggregate ceiling from an optional operator override. A missing or + * `null` override uses {@link DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES}. A present + * valid value passes through unchanged. + * + * A present invalid override does not fail host startup. The host process is + * multi-tenant, so one invalid environment value must not brick the whole host. + * The resolver rejects the invalid override, reports it through the optional + * `onRejectedOverride` reporter, and returns the conservative safe default. This + * fails loud without failing closed on availability. + */ +export function resolveDuplexAggregateCeilingBytes( + override?: number | null, + onRejectedOverride?: DuplexAggregateCeilingOverrideRejectionReporter, +): number { + if (override === undefined || override === null) { + return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; + } + if (!isValidDuplexAggregateCeilingBytes(override)) { + onRejectedOverride?.(override); + return DEFAULT_MAX_AGGREGATE_DUPLEX_ROUTE_BYTES; + } + return override; +} + +/** + * The process-owned aggregate byte ledger. Create one per host server process. + * Inject the same object into every host-side retention site, so one shared gauge + * bounds the aggregate retained bytes across all live duplex routes. + */ +export class DuplexAggregateByteLedger { + private readonly ceiling: number; + private readonly telemetry: DuplexAggregateByteLedgerTelemetry; + private used = 0; + /** The owner registry. It holds every live token. */ + private readonly liveTokens = new Set(); + + constructor(options: DuplexAggregateByteLedgerOptions) { + assertDuplexAggregateCeilingBytes(options.ceilingBytes); + this.ceiling = options.ceilingBytes; + this.telemetry = options.telemetry ?? NOOP_DUPLEX_AGGREGATE_BYTE_LEDGER_TELEMETRY; + } + + /** The configured aggregate ceiling, in bytes. */ + get ceilingBytes(): number { + return this.ceiling; + } + + /** The current aggregate bytes in use across every live token. */ + get bytesInUse(): number { + return this.used; + } + + /** The current number of live tokens in the owner registry. */ + get liveTokenCount(): number { + return this.liveTokens.size; + } + + /** + * Reserve `bytes` for `owner`. Check and increment the gauge synchronously + * before the caller allocates. Return a held token when the reservation fits + * under the ceiling. Return `null` when `used + bytes` would pass the ceiling; + * the caller retains nothing and reports + * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. + * + * The byte amount must be a non-negative safe integer. A caller that passes any + * other value made a programming error, so the ledger throws and fails loud. + */ + reserve(owner: DuplexAggregateTokenOwner, bytes: number): ReservationToken | null { + if (!Number.isSafeInteger(bytes) || bytes < 0) { + throw new Error( + `A duplex byte reservation must be a non-negative safe integer; got ${String(bytes)}.`, + ); + } + if (this.used + bytes > this.ceiling) { + this.telemetry.recordReservationRejection(); + return null; + } + this.used += bytes; + const token = new ReservationTokenImpl(bytes, owner); + this.liveTokens.add(token); + this.telemetry.setBytesInUse(this.used); + return token; + } + + /** + * Transfer a held token to a new owner label. Change only the owner. Never + * decrement and never re-reserve. This is the atomic pre-bind handoff: it moves + * the same reserved bytes from one representation to the next with no admission + * gap. + * + * A transfer of a token the ledger does not hold, or of an already-released + * token, is an accounting defect. The ledger records the fixed underflow + * counter and leaves the gauge unchanged. + */ + transfer(token: ReservationToken, owner: DuplexAggregateTokenOwner): void { + const impl = token as ReservationTokenImpl; + if (impl.state !== "held" || !this.liveTokens.has(impl)) { + this.telemetry.recordAccountingUnderflow(); + return; + } + impl.owner = owner; + } + + /** + * Release a held token. Remove it from the owner registry, mark it `released`, + * and decrement the gauge, all in one synchronous critical section before any + * `await` or external callback. A token releases one time. + * + * A second release of the same token, or a release of a token the ledger does + * not hold, is an accounting defect. The ledger records the fixed underflow + * counter and does not clamp the gauge a second time, so a real defect stays + * visible on the counter. + */ + release(token: ReservationToken): void { + const impl = token as ReservationTokenImpl; + if (impl.state === "released" || !this.liveTokens.delete(impl)) { + this.telemetry.recordAccountingUnderflow(); + return; + } + impl.state = "released"; + this.used -= impl.bytes; + this.telemetry.setBytesInUse(this.used); + } +} diff --git a/packages/adapter-utils/src/duplex-body-spool.test.ts b/packages/adapter-utils/src/duplex-body-spool.test.ts new file mode 100644 index 00000000000..37307f2e67e --- /dev/null +++ b/packages/adapter-utils/src/duplex-body-spool.test.ts @@ -0,0 +1,306 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { DUPLEX_FRAME_VERSION, type DuplexBodyChunkFrame } from "./duplex-frame-codec.js"; +import { + DuplexBodyReceiver, + type DuplexBodyReceiverConfig, + splitBodyIntoChunkFrames, + sweepStaleSpillDirectories, +} from "./duplex-body-spool.js"; + +// Read the whole reassembled body back, whatever path it took. A memory body +// returns its buffer; a spilled body streams from disk. The test uses this only +// to assert the round-trip content, never on a hot path. +async function readBodyBytes(body: { + spilled: boolean; + createReadStream: () => NodeJS.ReadableStream; +}): Promise { + const chunks: Buffer[] = []; + for await (const chunk of body.createReadStream()) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +let spillRoot: string; + +beforeEach(async () => { + spillRoot = await fs.mkdtemp(path.join(os.tmpdir(), "duplex-spool-test-")); +}); + +afterEach(async () => { + await fs.rm(spillRoot, { recursive: true, force: true }).catch(() => undefined); +}); + +function makeReceiver(overrides: DuplexBodyReceiverConfig = {}): Promise { + return DuplexBodyReceiver.create({ spillRootDir: spillRoot, ...overrides }); +} + +const RAW = 8; + +// Feed a whole body through a receiver as an envelope plus chunk frames. Return +// the settled body handle. The caller pushes the chunks the split produces. +async function transfer( + receiver: DuplexBodyReceiver, + id: string, + body: Buffer, +): ReturnType { + const promise = receiver.begin(id, body.length); + const frames = splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION, RAW); + for (const frame of frames) { + const result = receiver.pushChunk(frame); + expect(result.ok).toBe(true); + } + return promise; +} + +describe("duplex body reassembler — round trip", () => { + it("reassembles a small body on the memory path", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const source = Buffer.from("hello duplex body"); + const body = await transfer(receiver, "req-mem", source); + expect(body.spilled).toBe(false); + expect(body.byteCount).toBe(source.length); + expect(body.readMemory().equals(source)).toBe(true); + expect((await readBodyBytes(body)).equals(source)).toBe(true); + await body.dispose(); + }); + + it("reassembles a large body on the spill path without holding it in memory", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW }); + const source = Buffer.from("abcdefghijklmnopqrstuvwxyz0123456789ABCD"); // 40 bytes + const body = await transfer(receiver, "req-spill", source); + expect(body.spilled).toBe(true); + expect(body.byteCount).toBe(source.length); + // A spilled body never loads the whole body into memory on the receive side. + expect(() => body.readMemory()).toThrow(); + expect(receiver.spillUsedBytes).toBe(source.length); + expect(receiver.spillOpenFiles).toBe(1); + expect((await readBodyBytes(body)).equals(source)).toBe(true); + await body.dispose(); + // The dispose path unlinks the file and releases the reservation. + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); + + it("completes a zero-length body at once and accepts no chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 16, rawChunkBytes: RAW }); + const body = await receiver.begin("req-zero", 0); + expect(body.byteCount).toBe(0); + expect(body.readMemory().length).toBe(0); + const stray = receiver.pushChunk({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id: "req-zero", + seq: 0, + data: Buffer.from("x").toString("base64"), + }); + expect(stray.ok).toBe(false); + if (!stray.ok) expect(stray.error.code).toBe("zero_body_has_chunk"); + await body.dispose(); + }); +}); + +describe("duplex body reassembler — rejection matrix", () => { + function chunk(id: string, seq: number, data: string): DuplexBodyChunkFrame { + return { version: DUPLEX_FRAME_VERSION, type: "body_chunk", id, seq, data }; + } + + it("rejects a duplicate seq", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("d", 16); + expect(receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const dup = receiver.pushChunk(chunk("d", 0, Buffer.alloc(RAW, 2).toString("base64"))); + expect(dup.ok).toBe(false); + if (!dup.ok) expect(dup.error.code).toBe("seq_out_of_order"); + }); + + it("rejects a gap in the seq order", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("g", 24); + expect(receiver.pushChunk(chunk("g", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const gap = receiver.pushChunk(chunk("g", 2, Buffer.alloc(RAW, 2).toString("base64"))); + expect(gap.ok).toBe(false); + if (!gap.ok) expect(gap.error.code).toBe("seq_out_of_order"); + }); + + it("rejects a reordered seq", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("r", 24); + const reorder = receiver.pushChunk(chunk("r", 1, Buffer.alloc(RAW, 1).toString("base64"))); + expect(reorder.ok).toBe(false); + if (!reorder.ok) expect(reorder.error.code).toBe("seq_out_of_order"); + }); + + it("rejects non-canonical base64", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const begun = receiver.begin("b", 8); + begun.catch(() => undefined); + // "AB==" carries non-zero bits in the final sextet, so it decodes to one + // byte and re-encodes to "AA==". The round-trip mismatch marks it not + // canonical. + const bad = receiver.pushChunk(chunk("b", 0, "AB==")); + expect(bad.ok).toBe(false); + if (!bad.ok) expect(bad.error.code).toBe("chunk_not_canonical_base64"); + }); + + it("rejects a short non-final chunk (truncation)", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("t", 24); + // A non-final chunk must be exactly RAW bytes. A 4-byte chunk that does not + // complete the 24-byte body is a truncation. + const truncation = receiver.pushChunk(chunk("t", 0, Buffer.alloc(4, 1).toString("base64"))); + expect(truncation.ok).toBe(false); + if (!truncation.ok) expect(truncation.error.code).toBe("chunk_wrong_size"); + }); + + it("rejects an empty non-final chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("e", 16); + const empty = receiver.pushChunk(chunk("e", 0, "")); + expect(empty.ok).toBe(false); + if (!empty.ok) expect(empty.error.code).toBe("empty_chunk"); + }); + + it("rejects an overrun past the declared body size", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + // A 12-byte body takes one full RAW chunk, then a final chunk of 4 bytes. A + // second full RAW chunk overruns the declared size. + void receiver.begin("o", 12); + expect(receiver.pushChunk(chunk("o", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + const overrun = receiver.pushChunk(chunk("o", 1, Buffer.alloc(RAW, 2).toString("base64"))); + expect(overrun.ok).toBe(false); + if (!overrun.ok) expect(overrun.error.code).toBe("body_overrun"); + }); + + it("rejects an over-size final chunk", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + void receiver.begin("os", 40); + const over = receiver.pushChunk(chunk("os", 0, Buffer.alloc(RAW + 4, 1).toString("base64"))); + expect(over.ok).toBe(false); + if (!over.ok) expect(over.error.code).toBe("chunk_wrong_size"); + }); + + it("rejects a chunk that arrives with no matching envelope", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const orphan = receiver.pushChunk(chunk("missing", 0, Buffer.alloc(RAW, 1).toString("base64"))); + expect(orphan.ok).toBe(false); + if (!orphan.ok) expect(orphan.error.code).toBe("chunk_without_envelope"); + }); + + it("rejects a chunk that arrives after completion", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 1024, rawChunkBytes: RAW }); + const promise = receiver.begin("c", RAW); + expect(receiver.pushChunk(chunk("c", 0, Buffer.alloc(RAW, 1).toString("base64"))).ok).toBe(true); + await promise; + const late = receiver.pushChunk(chunk("c", 1, Buffer.alloc(RAW, 2).toString("base64"))); + expect(late.ok).toBe(false); + if (!late.ok) expect(late.error.code).toBe("chunk_after_completion"); + }); +}); + +describe("duplex body reassembler — spill caps fail closed", () => { + it("fails closed when a write exceeds the per-channel byte cap and unlinks the partial file", async () => { + // A 5-byte cap admits the first RAW=8 chunk write? No: reserve 8 > cap 5, so + // the first write fails closed. Use a cap that admits one chunk but not two. + const receiver = await makeReceiver({ + spillThresholdBytes: 4, + rawChunkBytes: RAW, + channelSpillByteCap: RAW + 2, + }); + const promise = receiver.begin("cap", 24); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // The second chunk write reserves past the byte cap. + receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "cap", seq: 1, data: Buffer.alloc(RAW, 2).toString("base64") }); + await expect(promise).rejects.toMatchObject({ code: "spill_cap_exceeded" }); + // The partial file is gone and the reservation released. + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); + + it("fails closed when the open-file cap is reached", async () => { + const receiver = await makeReceiver({ + spillThresholdBytes: 4, + rawChunkBytes: RAW, + channelSpillFileCap: 1, + }); + // The first spill body claims the one file slot and stays open. The channel + // dispose rejects this in-flight body, so attach the catch at once. + const first = receiver.begin("f1", 24); + first.catch(() => undefined); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "f1", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // The second spill body cannot claim a file slot. + await expect(receiver.begin("f2", 24)).rejects.toMatchObject({ code: "spill_cap_exceeded" }); + expect(receiver.spillOpenFiles).toBe(1); + // The dispose of the in-flight first body rejects its begin promise with a + // terminal channel-closed error and frees the file slot. + await receiver.disposeBody("f1"); + await expect(first).rejects.toMatchObject({ code: "channel_closed" }); + expect(receiver.spillOpenFiles).toBe(0); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — lifecycle cleanup", () => { + it("releases the reservation and unlinks the file on channel death", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW }); + // The channel death rejects this in-flight body, so attach the catch at once. + const dying = receiver.begin("death", 24); + dying.catch(() => undefined); + expect(receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "death", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }).ok).toBe(true); + // Wait for the write to land, so the file exists before the channel dies. + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(receiver.spillOpenFiles).toBe(1); + await receiver.disposeAll(); + await expect(dying).rejects.toMatchObject({ code: "channel_closed" }); + expect(receiver.spillUsedBytes).toBe(0); + expect(receiver.spillOpenFiles).toBe(0); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(0); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — spill file safety", () => { + it("creates the spill directory with mode 0700 and each spill file with mode 0600", async () => { + const receiver = await makeReceiver({ spillThresholdBytes: 4, rawChunkBytes: RAW }); + const dirStat = await fs.stat(receiver.spillDir); + expect(dirStat.mode & 0o777).toBe(0o700); + // The final destroy rejects this in-flight body, so attach the catch at once. + const pending = receiver.begin("mode", 8); + pending.catch(() => undefined); + receiver.pushChunk({ version: DUPLEX_FRAME_VERSION, type: "body_chunk", id: "mode", seq: 0, data: Buffer.alloc(RAW, 1).toString("base64") }); + await new Promise((resolve) => setTimeout(resolve, 20)); + const entries = await fs.readdir(receiver.spillDir); + expect(entries).toHaveLength(1); + // The file name is random, so no frame id rides the path. + expect(entries[0]).not.toContain("mode"); + const fileStat = await fs.stat(path.join(receiver.spillDir, entries[0])); + expect(fileStat.mode & 0o777).toBe(0o600); + await receiver.destroy(); + }); +}); + +describe("duplex body reassembler — startup sweep", () => { + it("removes a stale spill directory from a dead process but keeps a live one", async () => { + const deadPid = 999999; // A pid that is almost never alive. + const staleDir = path.join(spillRoot, `paperclip-duplex-spill-${deadPid}-stale`); + await fs.mkdir(staleDir); + await fs.writeFile(path.join(staleDir, "leftover.body"), "x"); + const liveDir = path.join(spillRoot, `paperclip-duplex-spill-${process.pid}-live`); + await fs.mkdir(liveDir); + await sweepStaleSpillDirectories(spillRoot); + await expect(fs.stat(staleDir)).rejects.toThrow(); + // The sweep keeps a directory that belongs to a live process. + await expect(fs.stat(liveDir)).resolves.toBeDefined(); + }); +}); diff --git a/packages/adapter-utils/src/duplex-body-spool.ts b/packages/adapter-utils/src/duplex-body-spool.ts new file mode 100644 index 00000000000..576caca5afe --- /dev/null +++ b/packages/adapter-utils/src/duplex-body-spool.ts @@ -0,0 +1,596 @@ +/** + * Receive-side body reassembly and spill for the sandbox duplex channel. + * + * The duplex codec splits one request body or one response body into an envelope + * frame plus a sequence of `body_chunk` frames. This module reassembles those + * chunks back into one body. The receiver reads `bodyByteCount` from the envelope + * first, then chooses one of two paths before it reads the first chunk: + * - memory path: a body at or below the spill threshold stays in a memory + * buffer. + * - spill path: a body above the spill threshold streams to a temporary file, + * so the receiver never holds a whole large body in memory. + * + * The spill path bounds the disk a single channel can consume. A per-channel + * arena caps the bytes on disk and the number of open spill files. The receiver + * reserves a file slot and reserves bytes before it opens a file and before each + * write. A reserve that exceeds a cap fails closed: it rejects the body as a + * resource error and unlinks any partial file. + * + * The reassembler treats every frame as untrusted. It rejects a bad chunk as a + * loud, terminal protocol error: a non-canonical base64 payload, a wrong chunk + * size, an out-of-order `seq`, a total that does not equal `bodyByteCount`, or a + * chunk that arrives for a completed or unknown body. + */ + +import { randomBytes } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { promises as fs } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { Readable } from "node:stream"; + +import { + DUPLEX_BODY_CHUNK_RAW_BYTES, + type DuplexBodyChunkFrame, +} from "./duplex-frame-codec.js"; + +/** + * The default spill threshold, in bytes. A body at or below this size stays in + * memory. A body above it spills to disk. The value is 1 MiB. It stays + * injectable so a test can lower it and exercise the spill path. + */ +export const DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES = 1024 * 1024; + +/** + * The default per-channel spill byte cap. The receiver keeps at most this many + * bytes on disk at one time for one channel. The value is 64 MiB. It is about + * 0.64% of the default 10 GiB sandbox disk and stays far below the security + * ceiling of 25% of the disk quota. It stays injectable so a test can lower it. + */ +export const DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP = 64 * 1024 * 1024; + +/** + * The default per-channel open-file cap. The receiver keeps at most this many + * spill files open at one time for one channel. The value derives from the byte + * cap and the spill threshold: 64 MiB divided by the 1 MiB spill floor gives 64. + * It stays injectable so a test can lower it. + */ +export const DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP = 64; + +/** The prefix of every spill directory name. The startup sweep matches it. */ +const SPILL_DIR_PREFIX = "paperclip-duplex-spill-"; + +/** The config a caller passes to build a {@link DuplexBodyReceiver}. */ +export interface DuplexBodyReceiverConfig { + /** The spill threshold in bytes. Defaults to {@link DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES}. */ + spillThresholdBytes?: number; + /** The fixed raw chunk size in bytes. Defaults to {@link DUPLEX_BODY_CHUNK_RAW_BYTES}. */ + rawChunkBytes?: number; + /** The per-channel spill byte cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP}. */ + channelSpillByteCap?: number; + /** The per-channel open-file cap. Defaults to {@link DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP}. */ + channelSpillFileCap?: number; + /** The root directory that holds the spill directory. Defaults to the OS temp dir. */ + spillRootDir?: string; + /** A diagnostic sink for a non-fatal cleanup note. The receiver names no body id here. */ + logger?: (message: string) => void; +} + +/** The reason the reassembler rejected a body. Each reason is a terminal protocol error. */ +export type DuplexBodyErrorCode = + | "chunk_without_envelope" + | "chunk_after_completion" + | "seq_out_of_order" + | "chunk_not_canonical_base64" + | "chunk_wrong_size" + | "empty_chunk" + | "body_overrun" + | "zero_body_has_chunk" + | "spill_cap_exceeded" + | "spill_write_failed" + | "path_escape" + | "channel_closed"; + +/** A reassembly error. The receiver returns it; it never leaks a raw path or a raw provider string. */ +export class DuplexBodyError extends Error { + readonly code: DuplexBodyErrorCode; + constructor(code: DuplexBodyErrorCode, message: string) { + super(message); + this.name = "DuplexBodyError"; + this.code = code; + } +} + +/** + * One reassembled body. The memory path holds the whole body in a buffer. The + * spill path holds a temporary file. A caller reads the memory buffer, or opens + * a fresh read stream, then disposes the body once. + */ +export interface ReassembledBody { + /** The raw byte length of the body. */ + readonly byteCount: number; + /** True when the body spilled to disk. */ + readonly spilled: boolean; + /** + * The whole body as a buffer. The memory path returns it. The spill path + * throws, because the spill path must never load a whole large body into + * memory on the receive side. A caller streams a spilled body instead. + */ + readMemory(): Buffer; + /** A fresh read stream over the body. The memory path streams one buffer; the spill path streams the file. */ + createReadStream(): Readable; + /** Release the body. The spill path unlinks the file and releases the reservation exactly once. */ + dispose(): Promise; +} + +/** + * The per-channel arena that bounds the spill disk use. It tracks the bytes on + * disk and the open files, and it owns the spill directory. It hands out + * reservations and takes them back. The reservation math is synchronous, so a + * concurrent reserve never races past a cap. + */ +class SpillArena { + usedBytes = 0; + openFiles = 0; + constructor( + readonly dir: string, + private readonly byteCap: number, + private readonly fileCap: number, + ) {} + + /** Reserve one open-file slot. Return false when the file cap is already at the limit. */ + reserveFile(): boolean { + if (this.openFiles >= this.fileCap) return false; + this.openFiles += 1; + return true; + } + + /** Release one open-file slot. */ + releaseFile(): void { + if (this.openFiles > 0) this.openFiles -= 1; + } + + /** Reserve `n` bytes of disk. Return false when the byte cap would be exceeded. */ + reserveBytes(n: number): boolean { + if (this.usedBytes + n > this.byteCap) return false; + this.usedBytes += n; + return true; + } + + /** Release `n` bytes of disk. */ + releaseBytes(n: number): void { + this.usedBytes -= n; + if (this.usedBytes < 0) this.usedBytes = 0; + } +} + +/** The internal state of one in-flight body. */ +interface ActiveBody { + bodyByteCount: number; + nextSeq: number; + receivedBytes: number; + spilled: boolean; + /** Memory-path accumulation. Empty for the spill path. */ + memoryChunks: Buffer[]; + /** The spill file path. Null for the memory path. */ + filePath: string | null; + /** The open spill file handle. Null for the memory path or before the first write. */ + handle: FileHandle | null; + /** The bytes this body reserved in the arena. The dispose path releases exactly this many. */ + reservedBytes: number; + /** True when the body holds an arena file slot that the dispose path must release. */ + holdsFileSlot: boolean; + /** True once the dispose path ran, so it runs its release once. */ + disposed: boolean; + /** Resolve the completion promise the caller awaits. */ + resolve: (body: ReassembledBody) => void; + /** Reject the completion promise on a terminal error. */ + reject: (error: DuplexBodyError) => void; + /** Serialize the spill writes, so the chunks land on disk in `seq` order. */ + writeChain: Promise; + /** True once the body settled (resolved or rejected), so a late chunk never double-settles. */ + settled: boolean; +} + +/** The synchronous result of one `pushChunk` call. */ +export type DuplexBodyPushResult = + | { ok: true } + | { ok: false; error: DuplexBodyError }; + +/** + * A per-channel receive-side body reassembler. One instance serves one channel. + * The caller feeds it one envelope per body, then the `body_chunk` frames for + * that body. The reassembler reassembles the body on the memory path or the + * spill path and settles one completion promise per body. + */ +export class DuplexBodyReceiver { + private readonly spillThresholdBytes: number; + private readonly rawChunkBytes: number; + private readonly arena: SpillArena; + private readonly logger?: (message: string) => void; + private readonly active = new Map(); + + private constructor(arena: SpillArena, config: DuplexBodyReceiverConfig) { + this.arena = arena; + this.spillThresholdBytes = + config.spillThresholdBytes ?? DEFAULT_DUPLEX_BODY_SPILL_THRESHOLD_BYTES; + this.rawChunkBytes = config.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; + this.logger = config.logger; + } + + /** + * Build a receiver. The receiver owns the spill directory. It creates the + * directory with `mkdtemp` and mode 0700, then sweeps stale spill directories + * from a prior process. The sweep runs and awaits before the receiver serves a + * body, so no caller races a half-swept directory. + */ + static async create(config: DuplexBodyReceiverConfig = {}): Promise { + const root = config.spillRootDir ?? os.tmpdir(); + await sweepStaleSpillDirectories(root, config.logger); + const dir = await fs.mkdtemp(path.join(root, `${SPILL_DIR_PREFIX}${process.pid}-`)); + // `mkdtemp` creates the directory with mode 0700 on a POSIX host by default, + // but a permissive umask can widen it. Set the mode so the owner alone reads, + // writes, and lists the spill directory. + await fs.chmod(dir, 0o700); + const arena = new SpillArena( + dir, + config.channelSpillByteCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_BYTE_CAP, + config.channelSpillFileCap ?? DEFAULT_DUPLEX_CHANNEL_SPILL_FILE_CAP, + ); + return new DuplexBodyReceiver(arena, config); + } + + /** The spill directory this receiver owns. A test reads it to assert containment and cleanup. */ + get spillDir(): string { + return this.arena.dir; + } + + /** The bytes on disk across all in-flight bodies for this channel. A test asserts the reservation. */ + get spillUsedBytes(): number { + return this.arena.usedBytes; + } + + /** The open spill files for this channel. A test asserts the file-slot reservation. */ + get spillOpenFiles(): number { + return this.arena.openFiles; + } + + /** + * Begin one body. The caller passes the envelope `id` and `bodyByteCount`. The + * receiver chooses the memory path or the spill path from `bodyByteCount` + * before the first chunk arrives. A `bodyByteCount` of 0 completes at once with + * an empty body and accepts no chunk. The returned promise resolves with the + * reassembled body, or rejects with a {@link DuplexBodyError} on a terminal + * protocol or resource error. + */ + begin(id: string, bodyByteCount: number): Promise { + return new Promise((resolve, reject) => { + // A repeated envelope id for an in-flight body is a protocol violation. The + // caller must not begin the same id twice. Reject the new begin and leave + // the in-flight body untouched. + if (this.active.has(id)) { + reject(new DuplexBodyError("chunk_after_completion", "duplicate body envelope id")); + return; + } + const body: ActiveBody = { + bodyByteCount, + nextSeq: 0, + receivedBytes: 0, + spilled: bodyByteCount > this.spillThresholdBytes, + memoryChunks: [], + filePath: null, + handle: null, + reservedBytes: 0, + holdsFileSlot: false, + disposed: false, + resolve, + reject, + writeChain: Promise.resolve(), + settled: false, + }; + this.active.set(id, body); + if (bodyByteCount === 0) { + // The zero-length body completes at once. It accepts no chunk, so the + // receiver keeps the id in a completed state to reject a stray chunk. + this.complete(id, body); + return; + } + if (body.spilled) { + // Reserve one file slot before the receiver opens the spill file. A + // reserve past the file cap fails closed at once. + if (!this.arena.reserveFile()) { + this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill file cap reached")); + return; + } + body.holdsFileSlot = true; + // Open the spill file with a random name and mode 0600. The name never + // derives from the frame id, so no id rides a path. Verify the resolved + // path stays inside the spill directory before the receiver writes. + this.openSpillFile(id, body); + } + }); + } + + /** + * Push one `body_chunk` frame. The receiver validates the chunk against the + * in-flight body, then appends it on the memory path or the spill path. The + * return value reports a synchronous validation outcome. A spill write failure + * or a completion rejects the begin promise instead. + */ + pushChunk(frame: DuplexBodyChunkFrame): DuplexBodyPushResult { + const body = this.active.get(frame.id); + if (!body) { + return fail("chunk_without_envelope", "body_chunk arrived with no matching envelope"); + } + if (body.bodyByteCount === 0) { + return fail("zero_body_has_chunk", "a zero-length body accepts no body_chunk"); + } + if (body.settled) { + return fail("chunk_after_completion", "body_chunk arrived after the body completed"); + } + if (frame.seq !== body.nextSeq) { + return fail("seq_out_of_order", "body_chunk seq is a duplicate, a gap, or a reorder"); + } + // Decode the base64 payload and confirm it is canonical. A non-canonical + // payload re-encodes to a different string, so the round-trip catches it. + const decoded = Buffer.from(frame.data, "base64"); + if (decoded.toString("base64") !== frame.data) { + return fail("chunk_not_canonical_base64", "body_chunk data is not canonical base64"); + } + if (decoded.length === 0) { + return fail("empty_chunk", "body_chunk is empty"); + } + const nextReceived = body.receivedBytes + decoded.length; + if (nextReceived > body.bodyByteCount) { + return fail("body_overrun", "body_chunk overruns the declared body size"); + } + const isFinal = nextReceived === body.bodyByteCount; + if (!isFinal && decoded.length !== this.rawChunkBytes) { + // A non-final chunk must be exactly the fixed raw slice size. A short + // non-final chunk is a truncation; an over-size chunk is a framing fault. + return fail("chunk_wrong_size", "a non-final body_chunk is not the fixed slice size"); + } + if (decoded.length > this.rawChunkBytes) { + // The final chunk must also stay within the fixed raw slice size. + return fail("chunk_wrong_size", "a body_chunk exceeds the fixed slice size"); + } + body.nextSeq += 1; + body.receivedBytes = nextReceived; + if (body.spilled) { + this.appendSpill(frame.id, body, decoded, isFinal); + } else { + body.memoryChunks.push(decoded); + if (isFinal) this.complete(frame.id, body); + } + return { ok: true }; + } + + /** + * Dispose one body by id, whatever its state. The dispose path unlinks the + * spill file and releases the arena reservation exactly once. The caller runs + * it on every settle path and on channel death, so a spill file never lingers. + */ + async disposeBody(id: string): Promise { + const body = this.active.get(id); + if (!body) return; + this.active.delete(id); + await this.release(body); + } + + /** + * Dispose every in-flight body. The caller runs it on channel death, so a + * channel that dies mid-body leaves no spill file behind. + */ + async disposeAll(): Promise { + const ids = [...this.active.keys()]; + await Promise.all(ids.map((id) => this.disposeBody(id))); + } + + /** + * Remove the spill directory this receiver owns. The caller runs it after it + * disposes every body, on channel shutdown. + */ + async destroy(): Promise { + await this.disposeAll(); + await fs.rm(this.arena.dir, { recursive: true, force: true }).catch(() => undefined); + } + + private openSpillFile(id: string, body: ActiveBody): void { + const name = `${randomBytes(24).toString("hex")}.body`; + const filePath = path.join(this.arena.dir, name); + // Path containment. The resolved file path must stay inside the spill + // directory. A generated random name never escapes, but the check is defense + // in depth against a future name scheme. + const resolvedDir = path.resolve(this.arena.dir); + const resolvedFile = path.resolve(filePath); + if (resolvedFile !== path.join(resolvedDir, name) || !resolvedFile.startsWith(resolvedDir + path.sep)) { + this.settleReject(id, body, new DuplexBodyError("path_escape", "spill path escapes the spill directory")); + return; + } + body.filePath = filePath; + // Open the file once, exclusive, mode 0600. The write chain appends each + // chunk in order to this one handle. + body.writeChain = body.writeChain.then(async () => { + body.handle = await fs.open(filePath, "wx", 0o600); + }); + body.writeChain.catch((error) => { + this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error))); + }); + } + + private appendSpill(id: string, body: ActiveBody, decoded: Buffer, isFinal: boolean): void { + // Reserve the bytes before the write. A reserve past the byte cap fails + // closed: reject the body and unlink the partial file. + if (!this.arena.reserveBytes(decoded.length)) { + this.settleReject(id, body, new DuplexBodyError("spill_cap_exceeded", "spill byte cap reached")); + return; + } + body.reservedBytes += decoded.length; + body.writeChain = body.writeChain.then(async () => { + if (body.settled) return; + if (!body.handle) throw new Error("spill file handle is not open"); + await body.handle.write(decoded); + if (isFinal) { + await body.handle.close(); + body.handle = null; + this.complete(id, body); + } + }); + body.writeChain.catch((error) => { + if (error instanceof DuplexBodyError) return; + this.settleReject(id, body, new DuplexBodyError("spill_write_failed", errorText(error))); + }); + } + + private complete(id: string, body: ActiveBody): void { + if (body.settled) return; + body.settled = true; + const receiver = this; + const handle: ReassembledBody = { + byteCount: body.bodyByteCount, + spilled: body.spilled, + readMemory(): Buffer { + if (body.spilled) { + throw new DuplexBodyError("spill_write_failed", "a spilled body has no in-memory buffer"); + } + return Buffer.concat(body.memoryChunks); + }, + createReadStream(): Readable { + if (body.spilled && body.filePath) return createReadStream(body.filePath); + return Readable.from(Buffer.concat(body.memoryChunks)); + }, + async dispose(): Promise { + await receiver.disposeBody(id); + }, + }; + body.resolve(handle); + } + + private settleReject(id: string, body: ActiveBody, error: DuplexBodyError): void { + if (body.settled) return; + body.settled = true; + // Release the reservation and unlink the partial file before the reject, so + // a rejected body never keeps a spill file or a reservation. The reject fires + // after the release, so a caller that awaits the reject sees a clean arena. + void this.disposeBody(id).finally(() => body.reject(error)); + } + + private async release(body: ActiveBody): Promise { + if (body.disposed) return; + body.disposed = true; + // Wait for the pending writes to settle, so the unlink never races an open + // handle. Swallow a write error here; the reject path already reported it. + await body.writeChain.catch(() => undefined); + if (body.handle) { + await body.handle.close().catch(() => undefined); + body.handle = null; + } + if (body.filePath) { + await fs.rm(body.filePath, { force: true }).catch((error) => { + this.logger?.(`Duplex spill unlink failed: ${errorText(error)}`); + }); + } + if (body.reservedBytes > 0) { + this.arena.releaseBytes(body.reservedBytes); + body.reservedBytes = 0; + } + if (body.holdsFileSlot) { + this.arena.releaseFile(); + body.holdsFileSlot = false; + } + // A body that the channel disposes before it settles must not leave its + // begin promise pending. Reject it with a terminal channel-closed error, so + // a caller that awaits the body sees the channel death. A body that already + // settled keeps its outcome: `complete` resolves it, and `settleReject` sets + // `settled` before it calls dispose, so this guard never double-settles. + if (!body.settled) { + body.settled = true; + body.reject( + new DuplexBodyError("channel_closed", "the channel closed before the body completed"), + ); + } + } +} + +function fail(code: DuplexBodyErrorCode, message: string): DuplexBodyPushResult { + return { ok: false, error: new DuplexBodyError(code, message) }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Split a whole body buffer into `body_chunk` frames. The send side reads the + * source body once, then calls this to build the frames. Each frame carries one + * raw slice of `rawChunkBytes` as base64 text, except the final frame, which + * carries the remaining bytes. A zero-length body yields no frame. + */ +export function splitBodyIntoChunkFrames( + id: string, + body: Buffer, + version: number, + rawChunkBytes: number = DUPLEX_BODY_CHUNK_RAW_BYTES, +): DuplexBodyChunkFrame[] { + const frames: DuplexBodyChunkFrame[] = []; + let seq = 0; + for (let offset = 0; offset < body.length; offset += rawChunkBytes) { + const slice = body.subarray(offset, offset + rawChunkBytes); + frames.push({ + version, + type: "body_chunk", + id, + seq, + data: slice.toString("base64"), + }); + seq += 1; + } + return frames; +} + +/** + * Sweep stale spill directories from a prior process. The receiver runs it at + * startup, before it serves a body. It removes a spill directory whose embedded + * pid is not this process and is not a live process, so a crash that left a + * directory behind does not leak disk. It never removes a directory of a live + * process, so a concurrent channel keeps its own spill directory. + */ +export async function sweepStaleSpillDirectories( + root: string, + logger?: (message: string) => void, +): Promise { + const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); + await Promise.all( + entries.map(async (entry) => { + if (!entry.isDirectory() || !entry.name.startsWith(SPILL_DIR_PREFIX)) return; + const pid = parsePidFromSpillDir(entry.name); + if (pid !== null && pid !== process.pid && isProcessAlive(pid)) return; + if (pid === process.pid) return; + await fs.rm(path.join(root, entry.name), { recursive: true, force: true }).catch((error) => { + logger?.(`Duplex stale spill sweep failed: ${errorText(error)}`); + }); + }), + ); +} + +/** Read the pid segment from a spill directory name, or null when it is absent. */ +function parsePidFromSpillDir(name: string): number | null { + const rest = name.slice(SPILL_DIR_PREFIX.length); + const dash = rest.indexOf("-"); + const pidText = dash === -1 ? rest : rest.slice(0, dash); + const pid = Number(pidText); + return Number.isInteger(pid) && pid > 0 ? pid : null; +} + +/** Return true when a process with the pid is alive. A signal 0 probes without a real signal. */ +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM means the process exists but this process cannot signal it. ESRCH + // means no such process. Treat only ESRCH as dead. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} diff --git a/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts new file mode 100644 index 00000000000..ff901dd1d98 --- /dev/null +++ b/packages/adapter-utils/src/duplex-bridge-broker-byte-ledger.test.ts @@ -0,0 +1,384 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import { + createDuplexBridgeBroker, + type DuplexBridgeBroker, + type DuplexBrokerForwardResult, +} from "./duplex-bridge-broker.js"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + DuplexAggregateByteLedger, +} from "./duplex-aggregate-byte-ledger.js"; +import { + DUPLEX_FRAME_VERSION, + DuplexFrameDecoder, + encodeDuplexFrame, + type DuplexRequestFrame, + type DuplexResponseFrame, +} from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; +import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; + +/** + * Regression harness for the broker aggregate byte ledger charging. + * + * The harness feeds request frames straight into the broker through an in-memory + * channel, the same as a provider that controls the transport. A test controls + * when each forward settles, so it can assert the ledger charge while the forward + * is still in flight and after it settles. + */ + +/** One pending forward a test settles by hand. */ +interface PendingForward { + id: string; + resolve: (result: DuplexBrokerForwardResult) => void; + reject: (error: Error) => void; + settled: boolean; +} + +/** + * One reassembled response the broker wrote back. The broker writes a response as + * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames + * that carry the body. The harness reassembles the body and exposes it as `body`. + */ +type ReassembledResponse = DuplexResponseFrame & { body: string }; + +/** One request the harness feeds: the envelope frame plus its raw body text. */ +interface RequestInput { + frame: DuplexRequestFrame; + bodyText: string; +} + +/** The in-memory channel plus the levers a test uses to drive the broker. */ +interface FakeChannelHarness { + channel: CommandManagedDuplexChannel; + feed: (input: RequestInput) => void; + exit: () => void; + responses: ReassembledResponse[]; + forwards: PendingForward[]; + resolveForward: (id: string, body?: string) => void; +} + +/** Build one valid request: the envelope carries `bodyByteCount`, the body rides body_chunk frames. */ +function requestFrame(id: string, method = "POST"): RequestInput { + const bodyText = JSON.stringify({ id }); + return { + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id, + method, + path: `/api/issues/${id}`, + query: "", + headers: { "content-type": "application/json" }, + bodyByteCount: Buffer.byteLength(bodyText, "utf8"), + }, + bodyText, + }; +} + +function createFakeChannelHarness(): FakeChannelHarness { + const responses: ReassembledResponse[] = []; + const forwards: PendingForward[] = []; + const writtenDecoder = new DuplexFrameDecoder(); + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); + let dataListener: ((chunk: string) => void) | null = null; + let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; + + const channel: CommandManagedDuplexChannel = { + write: (data) => { + for (const result of writtenDecoder.push(data)) { + if (!result.ok) continue; + const frame = result.frame; + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + responses.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) continue; + const decoded = Buffer.from(frame.data, "base64"); + assembly.received += decoded.length; + assembly.chunks.push(decoded); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + responses.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } + } + } + }, + onData: (listener) => { + dataListener = listener; + }, + onExit: (listener) => { + exitListener = listener; + }, + stop: () => undefined, + close: () => Promise.resolve(), + }; + + return { + channel, + feed: ({ frame, bodyText }) => { + if (!dataListener) throw new Error("The broker did not bind the data listener."); + dataListener(encodeDuplexFrame(frame)); + if (bodyText.length > 0) { + for (const chunk of splitBodyIntoChunkFrames( + frame.id, + Buffer.from(bodyText, "utf8"), + DUPLEX_FRAME_VERSION, + )) { + dataListener(encodeDuplexFrame(chunk)); + } + } + }, + exit: () => { + if (!exitListener) throw new Error("The broker did not bind the exit listener."); + exitListener({ exitCode: 0 }); + }, + responses, + forwards, + resolveForward: (id, body = JSON.stringify({ ok: true })) => { + const forward = [...forwards].reverse().find((entry) => entry.id === id && !entry.settled); + if (!forward) throw new Error(`No unsettled forward for id ${id}.`); + forward.settled = true; + forward.resolve({ status: 200, headers: { "content-type": "application/json" }, body }); + }, + }; +} + +/** A forward handler that hands each call to the harness and never auto-resolves. */ +function controllableForward(harness: FakeChannelHarness) { + return (request: DuplexRequestFrame): Promise => + new Promise((resolve, reject) => { + harness.forwards.push({ id: request.id, resolve, reject, settled: false }); + }); +} + +/** + * Wait for the microtasks and one macrotask to settle, so the broker reassembles + * a request body, dispatches its forward, and runs each settled promise handler. + */ +async function flush(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +describe("duplex bridge broker aggregate byte ledger", () => { + const brokers: DuplexBridgeBroker[] = []; + + afterEach(async () => { + while (brokers.length > 0) { + const broker = brokers.pop(); + if (broker) await broker.close(); + } + }); + + it("charges request-frame, request-payload, and seen-id tokens, then releases the request tokens on forward settlement", async () => { + const harness = createFakeChannelHarness(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + // The dispatch retains three tokens at the request envelope: the raw frame, the + // normalized payload, and the no-replay set entry. The reservation is + // synchronous, so the charge holds before the body reassembles. + expect(ledger.liveTokenCount).toBe(3); + expect(ledger.bytesInUse).toBeGreaterThan(0); + const chargedInFlight = ledger.bytesInUse; + + // The broker reassembles the request body, then dispatches the forward. + await flush(); + harness.resolveForward("req-1"); + await flush(); + + // The forward settled. The finally owner released the request-frame and the + // request-payload tokens. The seen-id token stays charged for the channel + // lifetime, so exactly one token remains. + expect(ledger.liveTokenCount).toBe(1); + expect(ledger.bytesInUse).toBeGreaterThan(0); + expect(ledger.bytesInUse).toBeLessThan(chargedInFlight); + + // A close releases the seen-id token, so the ledger returns to zero. + await broker.close(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("keeps request tokens charged when the response timer answers before the forward settles, and releases them on settlement", async () => { + const harness = createFakeChannelHarness(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + // Squeeze the nested budgets so the response timer fires quickly while the + // forward stays unsettled. + budgets: { forwardTimeoutMs: 5, responseBudgetMs: 10, gatewayWaitMs: 20 }, + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1", "GET")); + expect(ledger.liveTokenCount).toBe(3); + + // Wait past the response budget so the backstop answers the gateway while the + // forward is still in flight. + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(harness.responses.some((frame) => frame.id === "req-1")).toBe(true); + // The gateway got its answer, but the forward has not settled. The request + // tokens and the seen-id token stay charged: nothing released yet. + expect(ledger.liveTokenCount).toBe(3); + + // Settle the orphaned forward. Its finally owner now releases the two request + // tokens; the seen-id token still stays for the channel lifetime. + harness.resolveForward("req-1"); + await flush(); + expect(ledger.liveTokenCount).toBe(1); + + await broker.close(); + expect(ledger.bytesInUse).toBe(0); + }); + + it("refuses a one-byte-over dispatch with the fixed marker and retains nothing", async () => { + const harness = createFakeChannelHarness(); + // Measure the exact per-request charge with a throwaway broker, so the test + // can size a ceiling one byte below it and force the reservation to fail. + const probeHarness = createFakeChannelHarness(); + const measured = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const measuringBroker = await createDuplexBridgeBroker({ + channel: probeHarness.channel, + forwardRequest: controllableForward(probeHarness), + duplexAggregateByteLedger: measured, + }); + measuringBroker.start(); + probeHarness.feed(requestFrame("req-1")); + const perRequestBytes = measured.bytesInUse; + await measuringBroker.close(); + + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: perRequestBytes - 1 }); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + await flush(); + + // The broker retained nothing: no token, no forward, no seen id. + expect(ledger.liveTokenCount).toBe(0); + expect(ledger.bytesInUse).toBe(0); + expect(harness.forwards.length).toBe(0); + // The refusal carries the fixed marker and is a bounded terminal response. + const refusal = harness.responses.find((frame) => frame.id === "req-1"); + expect(refusal).toBeTruthy(); + expect(refusal?.status).toBe(503); + expect(refusal?.body).toContain(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + + // The refusal did not retain the id, so a resend after pressure eases is + // admitted: raise the ceiling and re-feed. + const roomyLedger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const roomyHarness = createFakeChannelHarness(); + const roomyBroker = await createDuplexBridgeBroker({ + channel: roomyHarness.channel, + forwardRequest: controllableForward(roomyHarness), + duplexAggregateByteLedger: roomyLedger, + }); + brokers.push(roomyBroker); + roomyBroker.start(); + roomyHarness.feed(requestFrame("req-1")); + await flush(); + expect(roomyHarness.forwards.length).toBe(1); + }); + + it("releases every retained token on a terminal channel loss", async () => { + const harness = createFakeChannelHarness(); + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1_000_000 }); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + harness.feed(requestFrame("req-2")); + // The reservation is synchronous, so both dispatches charge three tokens each + // at the request envelope, before the bodies reassemble. + expect(ledger.liveTokenCount).toBe(6); + + // Let both requests reassemble and start their forwards, so the loss below + // transfers live forwards to the orphan registry. + await flush(); + + // The channel exits mid-flight. The broker latches the loss, transfers the + // in-flight forwards to the orphan registry, and releases the seen-id tokens. + harness.exit(); + expect(broker.runDisposition.failed).toBe(true); + // The seen-id tokens released at loss; the two request tokens per forward stay + // charged until each aborted forward settles. + expect(ledger.liveTokenCount).toBe(4); + + // The aborted forwards settle. Each finally owner releases its two request + // tokens, so the ledger returns to zero. + harness.forwards[0]?.reject(new Error("aborted")); + harness.forwards[1]?.reject(new Error("aborted")); + await flush(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("does not double-release a token when a forward settles after the channel closed", async () => { + const harness = createFakeChannelHarness(); + let underflow = 0; + const ledger = new DuplexAggregateByteLedger({ + ceilingBytes: 1_000_000, + telemetry: { + setBytesInUse() {}, + recordReservationRejection() {}, + recordAccountingUnderflow() { + underflow += 1; + }, + }, + }); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + duplexAggregateByteLedger: ledger, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("req-1")); + // Let the request reassemble and start its forward before the close, so the + // close orphans a live forward that settles afterward. + await flush(); + await broker.close(); + // The close aborted the in-flight forward and orphaned it. The forward now + // settles after the close: its finally owner releases the request tokens once. + harness.forwards[0]?.reject(new Error("aborted")); + await flush(); + + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + // No accounting defect: every token released exactly one time. + expect(underflow).toBe(0); + }); +}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.test.ts b/packages/adapter-utils/src/duplex-bridge-broker.test.ts index d254742a269..346c0cbf392 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.test.ts @@ -14,8 +14,10 @@ import { type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import { createDuplexTelemetry, + DUPLEX_COUNTER_LOSS_TOTAL, DUPLEX_SPAN_REQUEST, type DuplexTelemetryCounterRecord, type DuplexTelemetryEventRecord, @@ -41,13 +43,27 @@ interface PendingForward { settled: boolean; } +/** + * One reassembled response the broker wrote back. The broker writes a response as + * one envelope frame that carries `bodyByteCount`, then the `body_chunk` frames + * that carry the body. The harness reassembles the body and exposes it as `body`, + * so a test asserts the response the same way it did with the one-frame model. + */ +type ReassembledResponse = DuplexResponseFrame & { body: string }; + +/** One request the harness feeds: the envelope frame plus its raw body text. */ +interface RequestInput { + frame: DuplexRequestFrame; + bodyText: string; +} + /** The in-memory channel plus the levers a test uses to drive the broker. */ interface FakeChannelHarness { channel: CommandManagedDuplexChannel; - /** Push one request frame into the broker read path. */ - feed: (frame: DuplexRequestFrame) => void; - /** The response frames the broker wrote back, in order. */ - responses: DuplexResponseFrame[]; + /** Push one request (its envelope frame and its body_chunk frames) into the broker read path. */ + feed: (input: RequestInput) => void; + /** The reassembled responses the broker wrote back, in order. */ + responses: ReassembledResponse[]; /** Every forward call the broker made, in order. */ forwards: PendingForward[]; /** Resolve the newest unsettled forward for one id with a 200 result. */ @@ -56,36 +72,71 @@ interface FakeChannelHarness { resolveAll: () => void; } -/** Build one valid request frame with distinctive, secret-looking fields. */ -function requestFrame(id: string, method = "POST"): DuplexRequestFrame { +/** + * Build one valid request with distinctive, secret-looking fields. The envelope + * carries `bodyByteCount`, and the harness rides the body on `body_chunk` frames. + */ +function requestFrame( + id: string, + method = "POST", + bodyText: string = JSON.stringify({ secret: "secret-request-body" }), +): RequestInput { return { - version: DUPLEX_FRAME_VERSION, - type: "request", - id, - method, - path: `/api/issues/${id}`, - query: "?secret-query=leak", - headers: { authorization: "Bearer super-secret-provider-token" }, - body: JSON.stringify({ secret: "secret-request-body" }), + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id, + method, + path: `/api/issues/${id}`, + query: "?secret-query=leak", + headers: { authorization: "Bearer super-secret-provider-token" }, + bodyByteCount: Buffer.byteLength(bodyText, "utf8"), + }, + bodyText, }; } /** * Build the in-memory channel harness. The channel keeps the broker read - * listener, so `feed` pushes an encoded frame into the broker. The channel - * decodes each written frame, so `responses` holds the response frames only. + * listener, so `feed` pushes the encoded envelope and its body_chunk frames into + * the broker. The channel decodes each written frame and reassembles a response + * body, so `responses` holds one reassembled response per delivered request. */ function createFakeChannelHarness(): FakeChannelHarness { - const responses: DuplexResponseFrame[] = []; + const responses: ReassembledResponse[] = []; const forwards: PendingForward[] = []; const writtenDecoder = new DuplexFrameDecoder(); + // The in-flight response reassembly, keyed by request id. + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); let dataListener: ((chunk: string) => void) | null = null; const channel: CommandManagedDuplexChannel = { write: (data) => { for (const result of writtenDecoder.push(data)) { - if (result.ok && result.frame.type === "response") { - responses.push(result.frame); + if (!result.ok) continue; + const frame = result.frame; + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + responses.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) continue; + const decoded = Buffer.from(frame.data, "base64"); + assembly.received += decoded.length; + assembly.chunks.push(decoded); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + responses.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } } } }, @@ -99,9 +150,17 @@ function createFakeChannelHarness(): FakeChannelHarness { return { channel, - feed: (frame) => { + feed: ({ frame, bodyText }) => { if (!dataListener) throw new Error("The broker did not bind the data listener."); dataListener(encodeDuplexFrame(frame)); + if (bodyText.length > 0) { + const chunks = splitBodyIntoChunkFrames( + frame.id, + Buffer.from(bodyText, "utf8"), + DUPLEX_FRAME_VERSION, + ); + for (const chunk of chunks) dataListener(encodeDuplexFrame(chunk)); + } }, responses, forwards, @@ -133,6 +192,11 @@ function controllableForward(harness: FakeChannelHarness) { }); } +/** Flush the microtasks and one macrotask, so the broker dispatches its reassembled forwards. */ +async function flush(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + /** The telemetry sink capture. It proves the broker records nothing for a refusal. */ interface TelemetryCapture { spans: DuplexTelemetrySpanRecord[]; @@ -174,11 +238,11 @@ describe("duplex bridge broker request limits", () => { expect(() => assertDuplexBrokerLimits({ maxInFlightRequests: 8, maxLifetimeRequests: 16 })).not.toThrow(); }); - it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", () => { + it("bounds the in-flight forwards and refuses the excess with a retryable terminal response", async () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const maxInFlightRequests = 4; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), telemetry, @@ -188,10 +252,13 @@ describe("duplex bridge broker request limits", () => { broker.start(); // Inject more than the limit of unique valid frames. Every forward hangs, so - // the broker holds the maximum number of in-flight forwards. + // the broker holds the maximum number of in-flight forwards. The broker gates + // the in-flight limit at the request envelope, so it refuses the excess at + // once; it dispatches each accepted forward after it reassembles the body. const total = 10; const ids = Array.from({ length: total }, (_unused, index) => `flight-${index}`); for (const id of ids) harness.feed(requestFrame(id)); + await flush(); // The broker forwarded only up to the limit. It refused the rest. expect(harness.forwards).toHaveLength(maxInFlightRequests); @@ -225,7 +292,7 @@ describe("duplex bridge broker request limits", () => { it("frees capacity after a delivered request and forwards a resent refused id (no-replay preserved)", async () => { const harness = createFakeChannelHarness(); const maxInFlightRequests = 2; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), maxInFlightRequests, @@ -233,30 +300,36 @@ describe("duplex bridge broker request limits", () => { brokers.push(broker); broker.start(); - // Saturate the in-flight limit with two hung forwards. + // Saturate the in-flight limit with two hung forwards. The broker reserves the + // in-flight slot at the request envelope, then dispatches the forward after it + // reassembles the body. harness.feed(requestFrame("keep-0")); harness.feed(requestFrame("keep-1")); + await flush(); expect(harness.forwards).toHaveLength(2); // A third unique id is refused, retryable. The broker did not forward it and // did not retain its id. harness.feed(requestFrame("resend-me")); + await flush(); expect(harness.forwards).toHaveLength(2); expect(harness.responses).toHaveLength(1); expect(JSON.parse(harness.responses[0].body).retryable).toBe(true); // Free one in-flight slot. The delivered request drains the pending record. harness.resolveForward("keep-0"); - await Promise.resolve(); + await flush(); // The gateway resends the refused id. Capacity is free now, so the broker // forwards it exactly one time. This proves the refusal did not poison the id. harness.feed(requestFrame("resend-me")); + await flush(); expect(harness.forwards.filter((forward) => forward.id === "resend-me")).toHaveLength(1); // A resend of an already-forwarded id never reaches the forward twice. const forwardedKeep1 = harness.forwards.filter((forward) => forward.id === "keep-1").length; harness.feed(requestFrame("keep-1")); + await flush(); expect(harness.forwards.filter((forward) => forward.id === "keep-1")).toHaveLength(forwardedKeep1); harness.resolveAll(); @@ -266,7 +339,7 @@ describe("duplex bridge broker request limits", () => { const harness = createFakeChannelHarness(); const { telemetry, capture } = createTelemetryCapture(); const maxLifetimeRequests = 3; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, // Resolve every forward at once, so each dispatched id leaves the in-flight // count but stays in the retained-id set. @@ -284,7 +357,7 @@ describe("duplex bridge broker request limits", () => { const total = 9; for (let index = 0; index < total; index += 1) { harness.feed(requestFrame(`life-${index}`)); - await Promise.resolve(); + await flush(); } // The broker forwarded only up to the lifetime limit. The retained-id set @@ -303,7 +376,7 @@ describe("duplex bridge broker request limits", () => { // A resend of a refused id stays refused, so the set never grows. harness.feed(requestFrame("life-8")); - await Promise.resolve(); + await flush(); expect(harness.forwards).toHaveLength(maxLifetimeRequests); // The telemetry surface stayed on the fixed request span for the delivered @@ -319,9 +392,9 @@ describe("duplex bridge broker request limits", () => { expect(broker.state).toBe("open"); }); - it("forwards nothing and fails the channel closed under a flood of over-limit ids", () => { + it("forwards nothing and fails the channel closed under a flood of over-limit ids", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -346,9 +419,9 @@ describe("duplex bridge broker request limits", () => { expect(broker.lossRecord?.reason).toBe("protocol_failure"); }); - it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", () => { + it("forwards a maximal-size id exactly once and does not forward a resend (no-replay preserved)", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -358,19 +431,21 @@ describe("duplex bridge broker request limits", () => { // An id at the maximum byte size is allowed. The broker forwards it one time. const maxId = "a".repeat(DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES); harness.feed(requestFrame(maxId)); + await flush(); expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); // Deliver the response, then resend the same id. The broker retained the id, // so the resend never reaches the forward a second time. harness.resolveForward(maxId); harness.feed(requestFrame(maxId)); + await flush(); expect(harness.forwards.filter((forward) => forward.id === maxId)).toHaveLength(1); expect(broker.state).toBe("open"); }); it("maps a forward that resolves with the indeterminate marker to a non-retryable response and retains the id", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -383,6 +458,7 @@ describe("duplex bridge broker request limits", () => { // non-retryable body, so the gateway maps it to a non-retryable status and a // caller never repeats a possibly-committed mutation. harness.feed(requestFrame("commit-1")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "commit-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -422,7 +498,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward rejection for a mutating method to a non-retryable indeterminate response and retains the id", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -435,6 +511,7 @@ describe("duplex bridge broker request limits", () => { // forward, so a POST must return a non-retryable indeterminate response. // A retryable status would let a caller repeat a committed mutation. harness.feed(requestFrame("mutate-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -461,9 +538,247 @@ describe("duplex bridge broker request limits", () => { expect(broker.state).toBe("open"); }); + it("substitutes a bounded terminal response for an oversized response and keeps the channel open", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + // A small frame bound makes a legitimately large response body exceed the + // bound, so the test exercises the encode guard without a megabyte body. + maxFrameBytes: 1000, + }); + brokers.push(broker); + broker.start(); + + // Two requests are in flight at the same time. One resolves with a normal + // body; the other resolves with a body that pushes the response frame over + // the bound. The host produced both results, so both requests reached the + // host. + harness.feed(requestFrame("small-1", "POST")); + harness.feed(requestFrame("big-1", "POST")); + await flush(); + + const smallForward = harness.forwards.find((entry) => entry.id === "small-1" && !entry.settled); + if (!smallForward) throw new Error("The broker did not forward the small request."); + smallForward.settled = true; + smallForward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ok: true }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + const bigForward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); + if (!bigForward) throw new Error("The broker did not forward the big request."); + bigForward.settled = true; + bigForward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ data: "y".repeat(2000) }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The other in-flight request delivered its full response, unaffected by the + // oversized one. + expect(harness.responses).toHaveLength(2); + const delivered = harness.responses.find((entry) => entry.id === "small-1")!; + expect(delivered.status).toBe(200); + expect(delivered.outcome).toBe("completed"); + expect(JSON.parse(delivered.body)).toEqual({ ok: true }); + + // The oversized response became one bounded terminal response marked + // indeterminate and non-retryable, so a caller does not retry a possible + // mutation. The broker did not write the oversized frame. + const terminal = harness.responses.find((entry) => entry.id === "big-1")!; + expect(terminal.status).toBe(502); + expect(terminal.outcome).toBe("indeterminate"); + expect(terminal.headers["x-paperclip-bridge-outcome"]).toBe("indeterminate"); + expect(JSON.parse(terminal.body)).toEqual({ + error: "upstream response too large to deliver", + outcome: "indeterminate", + retryable: false, + }); + + // The oversized body never crossed the wire. The broker did not truncate it or + // pass it through. + const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); + expect(writtenText).not.toContain("yyyy"); + + // The broker recorded the oversized request as an error outcome, never a loss. + // The channel stays open and the run does not fail. + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(1); + expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + + it("sends a minimal terminal response when the frame bound rejects the full replacement", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + // The bound accepts a compact request envelope (128 bytes) and the minimal + // terminal response envelope (114 bytes), but rejects the full replacement + // response envelope (193 bytes). This proves the broker never drops the channel + // or strands the request when the bound is smaller than the full replacement. + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + maxFrameBytes: 150, + }); + brokers.push(broker); + broker.start(); + + // A compact request envelope with an empty body fits the bound, so the broker + // forwards it. + harness.feed({ + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "big-1", + method: "POST", + path: "/api/issues/big-1", + query: "", + headers: {}, + bodyByteCount: 0, + }, + bodyText: "", + }); + await flush(); + + const forward = harness.forwards.find((entry) => entry.id === "big-1" && !entry.settled); + if (!forward) throw new Error("The broker did not forward the request."); + forward.settled = true; + // The host produced a large response body that pushes the real response frame + // and the full replacement frame over the bound. + forward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ data: "y".repeat(2000) }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The broker sent one minimal terminal response, not the oversized real + // response and not the full replacement. The minimal response carries the + // indeterminate outcome, so the gateway maps it to a terminal 409. + expect(harness.responses).toHaveLength(1); + const terminal = harness.responses[0]!; + expect(terminal.id).toBe("big-1"); + expect(terminal.status).toBe(502); + expect(terminal.outcome).toBe("indeterminate"); + expect(terminal.headers).toEqual({}); + expect(terminal.body).toBe(""); + + // The oversized body never crossed the wire. + const writtenText = harness.responses.map((entry) => encodeDuplexFrame(entry)).join(""); + expect(writtenText).not.toContain("yyyy"); + + // The broker recorded the request as an error outcome, never a loss. The + // channel stays open and the run does not fail. + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(1); + expect(errorSpans[0]!.name).toBe(DUPLEX_SPAN_REQUEST); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + + it("keeps the channel open and logs when the bound rejects even the minimal terminal response", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + const logs: string[] = []; + // The bound accepts a tiny request envelope (107 bytes) but rejects even the + // minimal terminal response envelope (110 bytes). The broker cannot send any + // frame for this request, so it must keep the channel open and log a clear + // local error. + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + maxFrameBytes: 109, + logger: (message) => logs.push(message), + }); + brokers.push(broker); + broker.start(); + + harness.feed({ + frame: { + version: DUPLEX_FRAME_VERSION, + type: "request", + id: "x", + method: "GET", + path: "/", + query: "", + headers: {}, + bodyByteCount: 0, + }, + bodyText: "", + }); + await flush(); + + const forward = harness.forwards.find((entry) => entry.id === "x" && !entry.settled); + if (!forward) throw new Error("The broker did not forward the request."); + forward.settled = true; + forward.resolve({ + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ data: "y".repeat(200) }), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The broker sent no response frame, because no frame fits the bound. + expect(harness.responses).toHaveLength(0); + // The broker recorded the request as an error outcome, never a loss. + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(1); + // The broker logged a clear local error and kept the channel open. + expect(logs.some((line) => line.includes("could not deliver a terminal response"))).toBe(true); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + + it("completes a request within the bound and delivers the real response", async () => { + const harness = createFakeChannelHarness(); + const { telemetry, capture } = createTelemetryCapture(); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: controllableForward(harness), + telemetry, + maxFrameBytes: 1000, + }); + brokers.push(broker); + broker.start(); + + harness.feed(requestFrame("ok-1", "POST")); + await flush(); + harness.resolveForward("ok-1", JSON.stringify({ ok: true })); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // The broker delivered the real response promptly with the completed outcome. + expect(harness.responses).toHaveLength(1); + const delivered = harness.responses[0]!; + expect(delivered.id).toBe("ok-1"); + expect(delivered.status).toBe(200); + expect(delivered.outcome).toBe("completed"); + expect(JSON.parse(delivered.body)).toEqual({ ok: true }); + + // The delivered response records an `ok` span, never an error, and never a loss. + const okSpans = capture.spans.filter((span) => span.dimensions.outcome === "ok"); + expect(okSpans).toHaveLength(1); + const errorSpans = capture.spans.filter((span) => span.dimensions.outcome === "error"); + expect(errorSpans).toHaveLength(0); + expect(broker.lossRecord).toBeNull(); + expect(broker.state).toBe("open"); + expect(broker.runDisposition.failed).toBe(false); + }); + it("maps a forward rejection for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), }); @@ -474,6 +789,7 @@ describe("duplex bridge broker request limits", () => { // mutation. A forward rejection for a GET stays retryable: the broker // returns a 502 with the completed outcome, so the gateway passes it through. harness.feed(requestFrame("read-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); forward.settled = true; @@ -492,7 +808,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward-budget timeout for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // A tiny forward budget aborts the controller before the manual rejection. @@ -505,6 +821,7 @@ describe("duplex bridge broker request limits", () => { // stay retryable: the broker returns a 504 with the completed outcome and no // indeterminate marker, so the gateway does not map it to a terminal 409. harness.feed(requestFrame("read-timeout-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-timeout-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -528,7 +845,7 @@ describe("duplex bridge broker request limits", () => { it("maps a forward-budget timeout for a mutating method to a non-retryable indeterminate response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), budgets: { forwardTimeoutMs: 5 }, @@ -540,6 +857,7 @@ describe("duplex bridge broker request limits", () => { // must keep the terminal contract: a 504 with the indeterminate outcome, so // the gateway maps it to a non-retryable 409 and no caller double-applies it. harness.feed(requestFrame("mutate-timeout-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-timeout-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -562,7 +880,7 @@ describe("duplex bridge broker request limits", () => { it("maps a response-budget backstop for a safe method to a retryable response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), // The forward budget aborts the call, but the forward promise stays @@ -579,6 +897,7 @@ describe("duplex bridge broker request limits", () => { // retryable: a 504 with the completed outcome and no indeterminate marker, // so the gateway does not map it to a terminal 409. harness.feed(requestFrame("read-stall-1", "GET")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "read-stall-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -599,7 +918,7 @@ describe("duplex bridge broker request limits", () => { it("maps a response-budget backstop for a mutating method to a non-retryable indeterminate response", async () => { const harness = createFakeChannelHarness(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: harness.channel, forwardRequest: controllableForward(harness), budgets: { forwardTimeoutMs: 5, responseBudgetMs: 15, gatewayWaitMs: 40 }, @@ -612,6 +931,7 @@ describe("duplex bridge broker request limits", () => { // contract: a 504 with the indeterminate outcome, so the gateway maps it to a // non-retryable 409 and no caller double-applies it. harness.feed(requestFrame("mutate-stall-1", "POST")); + await flush(); const forward = harness.forwards.find((entry) => entry.id === "mutate-stall-1" && !entry.settled); if (!forward) throw new Error("The broker did not forward the request."); @@ -629,3 +949,91 @@ describe("duplex bridge broker request limits", () => { }); }); }); + + +/** + * A channel harness that captures the broker exit listener. `emitExit` invokes it, + * so a test drives one channel exit into the broker. + */ +function createExitChannelHarness(): { + channel: CommandManagedDuplexChannel; + emitExit: (exit: { exitCode: number | null; transportClosed?: boolean }) => void; +} { + let exitListener: + | ((exit: { exitCode: number | null; transportClosed?: boolean }) => void) + | null = null; + const channel: CommandManagedDuplexChannel = { + write: () => undefined, + onData: () => undefined, + onExit: (listener) => { + exitListener = listener; + }, + stop: () => undefined, + close: () => Promise.resolve(), + }; + return { + channel, + emitExit: (exit) => { + if (!exitListener) throw new Error("The broker did not bind the exit listener."); + exitListener(exit); + }, + }; +} + +describe("duplex bridge broker exit taxonomy", () => { + const brokers: DuplexBridgeBroker[] = []; + + afterEach(async () => { + while (brokers.length > 0) { + const broker = brokers.pop(); + if (broker) await broker.close(); + } + }); + + it("records a transport close as a distinct loss, not a process exit", async () => { + const { telemetry, capture } = createTelemetryCapture(); + const harness = createExitChannelHarness(); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: () => + Promise.resolve({ status: 200, headers: {}, body: "" }), + telemetry, + }); + brokers.push(broker); + broker.start(); + + // A reason-less transport close carries the discriminator and no exit code. + harness.emitExit({ exitCode: null, transportClosed: true }); + + expect(broker.lossRecord?.reason).toBe("transport_closed"); + expect(broker.runDisposition).toEqual({ failed: true, lossReason: "transport_closed" }); + const lossCounter = capture.counters.find( + (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, + ); + expect(lossCounter?.dimensions.loss_reason).toBe("transport_closed"); + }); + + it("records a numeric exit as a process exit", async () => { + const { telemetry, capture } = createTelemetryCapture(); + const harness = createExitChannelHarness(); + const broker = await createDuplexBridgeBroker({ + channel: harness.channel, + forwardRequest: () => + Promise.resolve({ status: 200, headers: {}, body: "" }), + telemetry, + }); + brokers.push(broker); + broker.start(); + + // A numeric exit code is a real process exit, so the broker keeps the existing + // channel_exit -> provider_exit mapping. + harness.emitExit({ exitCode: 0 }); + + expect(broker.lossRecord?.reason).toBe("channel_exit"); + expect(broker.runDisposition).toEqual({ failed: true, lossReason: "provider_exit" }); + const lossCounter = capture.counters.find( + (record) => record.metric === DUPLEX_COUNTER_LOSS_TOTAL, + ); + expect(lossCounter?.dimensions.loss_reason).toBe("provider_exit"); + }); +}); diff --git a/packages/adapter-utils/src/duplex-bridge-broker.ts b/packages/adapter-utils/src/duplex-bridge-broker.ts index 2cba4097660..1ae50d92bb6 100644 --- a/packages/adapter-utils/src/duplex-bridge-broker.ts +++ b/packages/adapter-utils/src/duplex-bridge-broker.ts @@ -43,15 +43,31 @@ import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + type DuplexAggregateByteLedger, + type ReservationToken, +} from "./duplex-aggregate-byte-ledger.js"; +import { + DUPLEX_BODY_CHUNK_RAW_BYTES, + DEFAULT_MAX_DUPLEX_FRAME_BYTES, DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES, DUPLEX_FRAME_VERSION, DuplexFrameDecoder, encodeDuplexFrame, + encodeDuplexFrameChecked, + type DuplexBodyChunkFrame, type DuplexFrame, type DuplexRequestFrame, type DuplexResponseFrame, type DuplexResponseOutcome, } from "./duplex-frame-codec.js"; +import { + DuplexBodyError, + DuplexBodyReceiver, + splitBodyIntoChunkFrames, + type DuplexBodyReceiverConfig, + type ReassembledBody, +} from "./duplex-body-spool.js"; import type { DuplexLossReason, DuplexOutcomeValue, @@ -64,6 +80,7 @@ export type DuplexBrokerState = "opening" | "open" | "lost" | "closing" | "close /** The reason the broker classified a loss. The broker records it for metrics only. */ export type DuplexBrokerLossReason = | "channel_exit" + | "transport_closed" | "stream_failure" | "protocol_failure" | "heartbeat_write_failure" @@ -75,6 +92,8 @@ export type DuplexBrokerLossReason = * carries only the closed enum value. The map is total over the internal reasons, * so no raw text ever reaches the typed reason. * - `channel_exit` -> `provider_exit`: the provider channel process exited. + * - `transport_closed` -> `transport_closed`: the provider transport closed with + * no exit data, so the loss is a transport close, not a process exit. * - `stream_failure` -> `write_error`: a write to the channel failed. * - `protocol_failure` -> `rpc_failure`: a malformed or mismatched frame. * - `heartbeat_write_failure` -> `heartbeat_timeout`: the liveness write failed. @@ -82,6 +101,7 @@ export type DuplexBrokerLossReason = */ const BROKER_LOSS_REASON_TO_TYPED: Readonly> = { channel_exit: "provider_exit", + transport_closed: "transport_closed", stream_failure: "write_error", protocol_failure: "rpc_failure", heartbeat_write_failure: "heartbeat_timeout", @@ -163,6 +183,17 @@ export const DEFAULT_DUPLEX_BROKER_MAX_IN_FLIGHT_REQUESTS = 64; */ export const DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS = 50_000; +/** + * The fixed, documented per-entry allocation the broker charges the aggregate byte + * ledger for one no-replay request-id set entry. The type and the cardinality of + * `seenRequestIds` are bounded: the codec caps each id at + * {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}, and the lifetime limit caps the + * entry count. The broker charges the exact raw id bytes plus this fixed entry + * overhead, so the retained set never grows uncharged. This constant models the + * fixed per-entry cost of the string key and the Set slot, not the id bytes. + */ +export const DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES = 64; + /** The result of one forward call. The broker turns it into one response frame. */ export interface DuplexBrokerForwardResult { status: number; @@ -173,12 +204,15 @@ export interface DuplexBrokerForwardResult { /** * The forward handler the broker calls for each request. The handler applies the * real token and the run attribution, then forwards the request on the existing - * API path. The broker aborts `options.signal` when the forward budget ends or a - * loss happens, so a handler that threads the signal into its work stops early. + * API path. The broker reassembles the request body from the `body_chunk` frames + * before it calls the handler, so the handler streams the body from + * `options.body`. The broker aborts `options.signal` when the forward budget ends + * or a loss happens, so a handler that threads the signal into its work stops + * early. */ export type DuplexBrokerForwardHandler = ( request: DuplexRequestFrame, - options: { signal: AbortSignal }, + options: { signal: AbortSignal; body: ReassembledBody }, ) => Promise; /** One request record. The broker captures the dispatch-start point for metrics only. */ @@ -227,6 +261,14 @@ export interface DuplexBrokerOptions { * {@link DEFAULT_DUPLEX_BROKER_MAX_LIFETIME_REQUESTS}. */ maxLifetimeRequests?: number; + /** + * The config for the receive-side request-body reassembler. It sets the spill + * threshold, the fixed raw chunk size, and the per-channel spill caps. The + * broker creates one reassembler for the channel. The config stays injectable, + * so a test lowers the spill threshold and the caps to exercise the spill path + * and the fail-closed cap behavior without a large body. + */ + bodyReceiverConfig?: DuplexBodyReceiverConfig; /** The clock the broker reads for the metric timestamps. The default is `Date.now`. */ now?: () => number; /** The metrics sink for the per-request dispatch record. */ @@ -244,6 +286,17 @@ export interface DuplexBrokerOptions { * raw error rides a span or a counter. The default records nothing. */ telemetry?: DuplexTelemetry; + /** + * The process-owned aggregate byte ledger. The broker reserves the exact retained + * bytes of each dispatched request against it before it retains the frame: the + * raw request frame, the normalized request payload, and the no-replay set entry. + * A reservation that would pass the ceiling makes the broker retain nothing and + * refuse the request with the fixed marker + * {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. When the ledger is absent the + * broker charges nothing and behaves as before, so a non-duplex or a legacy path + * stays unchanged. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } /** The broker handle the factory returns. */ @@ -348,17 +401,56 @@ export function isSafeBridgeMethod(method: string): boolean { interface PendingRequest { controller: AbortController; responded: boolean; - forwardTimer: ReturnType; + /** + * The forward-budget timer. It is `null` while the broker reassembles the + * request body, and the broker sets it when it starts the forward. So the + * forward budget bounds the forward call, not the reassembly. + */ + forwardTimer: ReturnType | null; + /** + * The response-budget backstop timer. The broker starts it at the request + * envelope, so it bounds the whole request: the body reassembly plus the + * forward. It answers a request that never completes its reassembly or forward. + */ responseTimer: ReturnType; /** The point the broker started to dispatch the request. It sets the span latency. */ dispatchStartMs: number; + /** The request method. The response backstop reads it to classify the safe-method retry. */ + method: string; + /** The reassembled request body, set once reassembly completes. The broker disposes it on settle. */ + reassembled: ReassembledBody | null; + /** True once the forward promise settled. The finally owner sets it one time. */ + forwardSettled: boolean; + /** + * Release the request-frame token and the request-payload token exactly one time. + * The single forward-promise finally owner calls it after the forward settles. + * A second call is a no-op, so no token releases twice. + */ + releaseForwardTokens: () => void; } /** - * Create the host duplex bridge broker. The factory asserts the budget order and - * returns a handle. Call `start` to wire the channel and open the broker. + * One orphaned forward. The broker answered the gateway and removed the request + * from `pending`, but the forward promise or its response-body reader had not + * settled. The orphan keeps the request tokens charged and the controller live + * until the forward finally releases them, so the ledger reports nonzero ownership + * until the async work settles. */ -export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBridgeBroker { +interface OrphanedForward { + controller: AbortController; + /** Release the request-frame and request-payload tokens exactly one time. */ + releaseForwardTokens: () => void; +} + +/** + * Create the host duplex bridge broker. The factory asserts the budget order, + * creates the receive-side request-body reassembler, and returns a handle. It is + * asynchronous because the reassembler owns a spill directory it creates with + * `mkdtemp`. Call `start` to wire the channel and open the broker. + */ +export async function createDuplexBridgeBroker( + options: DuplexBrokerOptions, +): Promise { const budgets: DuplexBrokerBudgets = { ...DEFAULT_DUPLEX_BROKER_BUDGETS, ...options.budgets, @@ -376,9 +468,71 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_DUPLEX_BROKER_HEARTBEAT_INTERVAL_MS; const closeTimeoutMs = options.closeTimeoutMs ?? DEFAULT_DUPLEX_BROKER_CLOSE_TIMEOUT_MS; const now = options.now ?? (() => Date.now()); - const decoder = new DuplexFrameDecoder( - options.maxFrameBytes ? { maxFrameBytes: options.maxFrameBytes } : undefined, - ); + + // The process-owned aggregate byte ledger, or `null` when the caller injected + // none. When `null` the broker charges nothing and behaves as before. + const ledger = options.duplexAggregateByteLedger ?? null; + // The one frame size bound the broker enforces on both sides. The decoder + // rejects an inbound frame over this bound, and the encode guard refuses to + // write an outbound frame over it. Encode and decode share one value, so a + // frame the broker writes always decodes on the peer. + const maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES; + // The decoder charges its retained partial-frame bytes against the same host + // ledger. It receives the ledger object directly, so one gauge bounds the + // decoder buffer with every other host retention site. + const decoder = new DuplexFrameDecoder({ + maxFrameBytes, + ...(ledger ? { aggregateByteLedger: ledger } : {}), + }); + // Release one token exactly one time. A `null` token or an absent ledger is a + // no-op, so the broker never records a false accounting defect. + const releaseToken = (token: ReservationToken | null): void => { + if (ledger && token) ledger.release(token); + }; + // The byte size of the normalized request payload the broker retains across the + // forward. It counts the exact retained scalar and header bytes plus the raw body + // byte count, so the charge matches the reassembled body the broker holds and + // never a parsed object graph. The body rides `body_chunk` frames, so the + // envelope carries only `bodyByteCount`; that count is the retained body size. + const requestPayloadBytes = (frame: DuplexRequestFrame): number => { + let bytes = + Buffer.byteLength(frame.id, "utf8") + + Buffer.byteLength(frame.method, "utf8") + + Buffer.byteLength(frame.path, "utf8") + + Buffer.byteLength(frame.query, "utf8") + + frame.bodyByteCount; + for (const [key, value] of Object.entries(frame.headers)) { + bytes += Buffer.byteLength(key, "utf8") + Buffer.byteLength(value, "utf8"); + } + return bytes; + }; + + // The receive-side request-body reassembler. It reassembles each request body + // from the `body_chunk` frames, on the memory path at or below the spill + // threshold and on the spill path above it, so the broker never holds a whole + // large request body in memory. It owns a per-channel spill directory and the + // spill caps. + const receiver = await DuplexBodyReceiver.create(options.bodyReceiverConfig ?? {}); + let receiverFinalized = false; + const finalizeReceiver = (): Promise => { + // Remove the spill directory and every in-flight body once. The broker calls + // it on every terminal path: a loss, a normal teardown, and an orderly close. + if (receiverFinalized) return Promise.resolve(); + receiverFinalized = true; + return receiver.destroy().catch(() => undefined); + }; + // The fixed raw slice size for the response `body_chunk` frames. It matches the + // reassembler config, so a test that lowers the slice size splits both a request + // body and a response body the same way. + const rawChunkBytes = options.bodyReceiverConfig?.rawChunkBytes ?? DUPLEX_BODY_CHUNK_RAW_BYTES; + // The ids the broker is reassembling right now. A `body_chunk` for an id in this + // set routes to the reassembler. The broker removes an id when the body settles. + const reassembling = new Set(); + // The ids the broker refused at the envelope, with the raw bytes it still must + // drain. The sender emits the `body_chunk` frames for a refused request before + // it learns of the refusal, so the broker drains and drops those chunks instead + // of treating them as a body_chunk with no envelope. + const draining = new Map(); let state: DuplexBrokerState = "opening"; let stopped = false; @@ -423,7 +577,24 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // The ids the broker already dispatched. The broker forwards one id one time, // so a repeated frame never reaches the API twice. const seenRequestIds = new Set(); + // The aggregate-ledger tokens for the no-replay set entries. The broker holds + // one token per live set entry and releases every token one time at terminal + // teardown, so the retained set never leaves bytes charged after the channel + // ends. + const seenRequestIdTokens = new Set(); const pending = new Map(); + // The forwards that answered the gateway but whose async work has not settled. + // The broker keeps their tokens charged and their controller live until each + // forward finally releases its tokens, so the ledger reports the real ownership. + const orphanedForwards = new Map(); + + // Release every no-replay set-entry token one time and clear the set. The + // ledger release is one way, and the cleared set stops any second release, so + // this helper is safe to call more than one time at terminal teardown. + const releaseSeenRequestIdTokens = (): void => { + for (const token of seenRequestIdTokens) releaseToken(token); + seenRequestIdTokens.clear(); + }; const setState = (next: DuplexBrokerState): void => { if (state === next) return; @@ -439,12 +610,39 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr }; const clearPending = (): void => { - for (const entry of pending.values()) { - clearTimeout(entry.forwardTimer); + for (const [id, entry] of pending) { + if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); clearTimeout(entry.responseTimer); entry.controller.abort(new Error("Duplex broker stopped.")); + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + if (entry.forwardSettled) continue; + if (entry.forwardTimer !== null) { + // A live forward promise still owns its request tokens. Transfer it to the + // orphan registry, so its tokens stay charged until the forward finally + // releases them. The abort only asks the forward to stop; it never releases + // a token by itself. + orphanedForwards.set(id, { + controller: entry.controller, + releaseForwardTokens: entry.releaseForwardTokens, + }); + } else { + // The request is still reassembling, so no forward promise will settle and + // release its tokens. Release the request tokens now. The reassembler + // teardown below rejects the in-flight body and unlinks any spill file. + entry.releaseForwardTokens(); + } } pending.clear(); + reassembling.clear(); + draining.clear(); + // Ask every already-orphaned forward to stop as well. The forward-promise + // finally owner releases each orphan token when the forward settles. + for (const orphan of orphanedForwards.values()) { + orphan.controller.abort(new Error("Duplex broker stopped.")); + } }; const recordLoss = (reason: DuplexBrokerLossReason, message: string): void => { @@ -452,13 +650,18 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr if (stopped) return; const afterOrderlyCompletion = orderlyCompletionSeq !== null; // A clean channel end that orders after a host-observed orderly completion is - // a normal teardown, not a loss. Stop cleanly, emit no loss event, and leave - // the run a success. This keeps the closed telemetry contract: an orderly - // close is not a loss and emits no loss event. - if (afterOrderlyCompletion && reason === "channel_exit") { + // a normal teardown, not a loss. A process exit and a reason-less transport + // close both end the channel, so both count as the normal teardown here. Stop + // cleanly, emit no loss event, and leave the run a success. This keeps the + // closed telemetry contract: an orderly close is not a loss and emits no loss + // event. + if (afterOrderlyCompletion && (reason === "channel_exit" || reason === "transport_closed")) { stopped = true; clearHeartbeat(); clearPending(); + releaseSeenRequestIdTokens(); + decoder.dispose(); + void finalizeReceiver(); if (state !== "closing") setState("closed"); return; } @@ -482,6 +685,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } clearHeartbeat(); clearPending(); + releaseSeenRequestIdTokens(); + decoder.dispose(); + void finalizeReceiver(); lossRecord = { reason, message, atMs: now() }; setState("lost"); // Log the internal reason only. The broker never writes the raw provider @@ -491,9 +697,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr options.telemetry?.recordLoss(lossClass, BROKER_LOSS_REASON_TO_TYPED[reason] ?? "other"); }; - const writeFrame = (frame: DuplexFrame): boolean => { + const writeLine = (line: string): boolean => { try { - channel.write(encodeDuplexFrame(frame)); + channel.write(line); return true; } catch (error) { recordLoss("stream_failure", errorMessage(error)); @@ -501,6 +707,98 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } }; + // The result of one bounded send. `sent` means every frame went out; `too_large` + // means the response envelope exceeds the bound and the broker wrote nothing; + // `lost` means a write failed and the broker recorded the channel loss. + type SendResult = "sent" | "too_large" | "lost"; + + // Send one full response as a sequence of frames: one envelope frame that + // carries `bodyByteCount`, then the `body_chunk` frames that carry the body. The + // envelope is always small. Each `body_chunk` carries one fixed raw slice, so + // each encoded chunk stays under the frame bound by construction. The result + // reports `too_large` when the envelope itself exceeds the bound and the broker + // wrote nothing, `lost` when a write failed and the broker recorded the channel + // loss, and `sent` when every frame went out. + const sendResponseFrames = ( + id: string, + status: number, + headers: Record, + bodyText: string, + outcome: DuplexResponseOutcome, + ): SendResult => { + const bodyBuffer = Buffer.from(bodyText, "utf8"); + const envelope: DuplexResponseFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status, + headers, + bodyByteCount: bodyBuffer.length, + outcome, + }; + // Guard the envelope against the frame bound. The envelope holds no body, so a + // real bound rejects it only under an extreme small test bound. Report the + // rejection without a channel loss, so the caller decides how to answer. + const encodedEnvelope = encodeDuplexFrameChecked(envelope, maxFrameBytes); + if (!encodedEnvelope.ok) return "too_large"; + const chunkFrames = splitBodyIntoChunkFrames(id, bodyBuffer, DUPLEX_FRAME_VERSION, rawChunkBytes); + if (!writeLine(encodedEnvelope.line)) return "lost"; + for (const chunk of chunkFrames) { + const encodedChunk = encodeDuplexFrameChecked(chunk, maxFrameBytes); + if (!encodedChunk.ok) { + // A single fixed-size slice never exceeds the bound in a real configuration. + // A test bound below one chunk can reject it. The envelope already went out + // with the true `bodyByteCount`, so the broker cannot complete the body + // within the bound. Log the drop and stop; the gateway ends the request on + // its wait budget. Report `sent`, because the broker already committed the + // envelope, so no caller resends a bounded replacement over the same id. + options.logger?.("Duplex broker dropped an oversized body_chunk frame."); + return "sent"; + } + if (!writeLine(encodedChunk.line)) return "lost"; + } + return "sent"; + }; + + const sendTerminalIndeterminate = (id: string): void => { + // Answer one request the broker cannot deliver with the real result. The + // request reached the host and may have changed state, so the response is + // non-retryable and carries the `indeterminate` outcome. The broker tries the + // full replacement first. When the frame bound rejects even the full + // replacement envelope, the broker sends a minimal replacement that carries + // only the `indeterminate` outcome, so the gateway still ends the request and + // never waits for its full wait budget. When the bound rejects even the + // minimal replacement envelope, the broker logs a clear local error and keeps + // the channel open; it records no channel loss. + const fullBody = JSON.stringify({ + error: "upstream response too large to deliver", + outcome: "indeterminate", + retryable: false, + }); + if ( + sendResponseFrames( + id, + 502, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "indeterminate" }, + fullBody, + "indeterminate", + ) !== "too_large" + ) { + return; + } + // The bound rejects the full replacement envelope. Send a minimal terminal + // response with empty headers and an empty body. The `indeterminate` outcome + // still rides the envelope, so the gateway maps the request to a terminal 409. + if (sendResponseFrames(id, 502, {}, "", "indeterminate") !== "too_large") return; + // The bound rejects even the minimal terminal envelope. The broker cannot + // deliver any frame for this request within the bound. Log a clear local + // error and keep the channel open for every other request. The gateway ends + // its own outstanding request on its wait budget. + options.logger?.( + `Duplex broker could not deliver a terminal response within the ${maxFrameBytes}-byte frame bound.`, + ); + }; + const respond = ( id: string, result: DuplexBrokerForwardResult, @@ -510,28 +808,35 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr const entry = pending.get(id); if (!entry || entry.responded) return; entry.responded = true; - clearTimeout(entry.forwardTimer); - clearTimeout(entry.responseTimer); - pending.delete(id); + settlePendingBookkeeping(id, entry); // Do not write on a lost or closed channel. The gateway answers its own // outstanding request on loss, so a late write would go to a dead channel. if (state !== "open") return; + const bodyText = result.body ?? ""; + const bodyByteCount = Buffer.byteLength(bodyText, "utf8"); + // The response body rides `body_chunk` frames, so a large body no longer makes + // one frame too large. The gateway reassembles a response body in memory, + // though, so a body over the frame bound would force the gateway to hold an + // oversized body in memory. Treat a response body over the frame bound like the + // former oversized case: send a bounded, non-retryable indeterminate terminal + // response, and record the request outcome as an error, never a loss. + if (bodyByteCount > maxFrameBytes) { + options.telemetry?.recordRequest({ + latencyMs: now() - entry.dispatchStartMs, + outcome: "error", + }); + sendTerminalIndeterminate(id); + return; + } // Record the request span for the delivered request. The span carries the - // latency and the outcome only; no route, query, body, or token rides it. + // latency and the outcome only; no route, query, body, or token rides it. The + // broker records the span before the writes, so the outcome is recorded even if + // a later chunk write records a channel loss. options.telemetry?.recordRequest({ latencyMs: now() - entry.dispatchStartMs, outcome: telemetryOutcome, }); - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", - id, - status: result.status, - headers: result.headers ?? {}, - body: result.body ?? "", - outcome, - }; - writeFrame(frame); + sendResponseFrames(id, result.status, result.headers ?? {}, bodyText, outcome); }; const respondSaturated = (id: string, retryable: boolean): void => { @@ -541,23 +846,83 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // it holds only the fixed error shape. The `unavailable` outcome tells the // gateway this is not a delivered host response, so it never counts as one. if (state !== "open") return; - const frame: DuplexResponseFrame = { - version: DUPLEX_FRAME_VERSION, - type: "response", + sendResponseFrames( id, - status: 503, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "unavailable", - }, - body: JSON.stringify({ + 503, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, + JSON.stringify({ error: "Duplex broker capacity limit reached.", outcome: "unavailable", retryable, }), - outcome: "unavailable", - }; - writeFrame(frame); + "unavailable", + ); + }; + + // Release the timers and route the reassembly and ledger resources for one + // request the broker just answered. The broker calls it exactly once per + // request, from the response path. It clears both timers and drops the pending + // record, then it settles the resources by the request phase: + // - A live forward promise (the response backstop answered while the forward + // ran) keeps its request tokens and its reassembled body until its finally + // owner releases and disposes them, so the orphan registry holds it. + // - A request still reassembling (the backstop answered before the forward + // started) has no forward promise to settle, so the broker releases its + // tokens and disposes the in-flight body now. + // - A settled forward already released and disposed through its finally owner. + const settlePendingBookkeeping = (id: string, entry: PendingRequest): void => { + if (entry.forwardTimer !== null) clearTimeout(entry.forwardTimer); + clearTimeout(entry.responseTimer); + pending.delete(id); + if (entry.forwardSettled) return; + if (entry.forwardTimer !== null) { + // A live forward still owns its request tokens and streams the reassembled + // body. Keep both charged until the forward finally owner settles them. + orphanedForwards.set(id, { + controller: entry.controller, + releaseForwardTokens: entry.releaseForwardTokens, + }); + return; + } + // The request is still reassembling, so no forward promise will settle. Release + // the request tokens and dispose the in-flight body, so no spill file, arena + // reservation, or ledger token lingers. + entry.releaseForwardTokens(); + if (reassembling.delete(id)) void receiver.disposeBody(id); + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + }; + + // Mark a refused or duplicate request id for chunk draining. The sender emits + // the `body_chunk` frames for the request before it learns of the refusal, so + // the broker records the raw byte count it must drain and drop. A request with + // no body needs no draining, so the broker records only a non-zero count. + const markDrain = (frame: DuplexRequestFrame): void => { + if (frame.bodyByteCount > 0) draining.set(frame.id, frame.bodyByteCount); + }; + + const respondAggregateExceeded = (id: string): void => { + // Answer a request the aggregate byte ledger refused with a bounded terminal + // response. The broker reserved nothing, added no id to the seen set, and made + // no controller, timer, or forward, so the host API stays untouched. The + // response carries the fixed marker only; it holds no route, query, body, or + // token. The `unavailable` outcome tells the gateway this is not a delivered + // host response. The refusal is retryable, because the broker did not retain + // the id: the aggregate pressure can ease, and a resend can then get through. + if (state !== "open") return; + sendResponseFrames( + id, + 503, + { "content-type": "application/json", "x-paperclip-bridge-outcome": "unavailable" }, + JSON.stringify({ + error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + outcome: "unavailable", + retryable: true, + }), + "unavailable", + ); }; const dispatch = (frame: DuplexRequestFrame): void => { @@ -570,13 +935,19 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // forwards. It answers with the bounded terminal refusal, which carries no // route, query, body, or token, and records no telemetry. The refusal is not // retryable, because a resend of the same over-limit id never gets past this - // bound. + // bound. It drains the following chunks of the refused body. if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { respondSaturated(frame.id, false); + markDrain(frame); + return; + } + // Forward one id one time. A repeated id never reaches the API twice. The + // broker already answered the first request, so it answers no second time; it + // drains the resent body chunks and drops them. + if (seenRequestIds.has(frame.id)) { + markDrain(frame); return; } - // Forward one id one time. A repeated id never reaches the API twice. - if (seenRequestIds.has(frame.id)) return; // Bound the retained request-id memory. The broker keeps one id per distinct // dispatched request for the no-replay guarantee, so the set can only grow. // Once the broker reaches the lifetime limit, it refuses each new distinct id @@ -585,6 +956,7 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // the set never grows past the limit. if (seenRequestIds.size >= limits.maxLifetimeRequests) { respondSaturated(frame.id, false); + markDrain(frame); return; } // Bound the in-flight request count. Once the broker holds the maximum number @@ -592,12 +964,64 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr // it. The broker does not add the id to the seen set, so the gateway can // resend the request after an in-flight request completes. The refusal is // retryable for that reason. This check bounds the controllers, the timers, - // and the concurrent forwards a provider can force. + // the concurrent reassemblies, and the concurrent forwards a provider can + // force. if (pending.size >= limits.maxInFlightRequests) { respondSaturated(frame.id, true); + markDrain(frame); return; } + // Reserve the exact retained bytes against the aggregate ledger before the + // broker retains anything. It reserves three tokens in order: the raw request + // frame, the normalized request payload, and the no-replay set entry. A + // reservation that would pass the ceiling makes the broker retain nothing, + // release the tokens it already took, and refuse the request with the fixed + // marker. The broker adds no id to the seen set and makes no controller, timer, + // or forward, so the host API stays untouched. When the ledger is absent all + // tokens stay `null` and the broker behaves as before. + let requestFrameToken: ReservationToken | null = null; + let requestPayloadToken: ReservationToken | null = null; + let seenRequestIdToken: ReservationToken | null = null; + if (ledger) { + const rawFrameBytes = Buffer.byteLength(encodeDuplexFrame(frame), "utf8"); + requestFrameToken = ledger.reserve("request_frame", rawFrameBytes); + if (!requestFrameToken) { + respondAggregateExceeded(frame.id); + markDrain(frame); + return; + } + requestPayloadToken = ledger.reserve("request_payload", requestPayloadBytes(frame)); + if (!requestPayloadToken) { + ledger.release(requestFrameToken); + respondAggregateExceeded(frame.id); + markDrain(frame); + return; + } + const seenEntryBytes = + Buffer.byteLength(frame.id, "utf8") + DUPLEX_SEEN_REQUEST_ID_SET_ENTRY_BYTES; + seenRequestIdToken = ledger.reserve("seen_request_id", seenEntryBytes); + if (!seenRequestIdToken) { + ledger.release(requestFrameToken); + ledger.release(requestPayloadToken); + respondAggregateExceeded(frame.id); + markDrain(frame); + return; + } + } seenRequestIds.add(frame.id); + if (seenRequestIdToken) seenRequestIdTokens.add(seenRequestIdToken); + + // Release the request-frame and the request-payload tokens exactly one time. + // The single forward-promise finally owner calls it after the forward settles. + // The seen-id token stays charged for the channel lifetime, so it is not part + // of this release; the terminal teardown releases it. + let forwardTokensReleased = false; + const releaseForwardTokens = (): void => { + if (forwardTokensReleased) return; + forwardTokensReleased = true; + releaseToken(requestFrameToken); + releaseToken(requestPayloadToken); + }; const record: DuplexBrokerRequestRecord = { id: frame.id, @@ -608,84 +1032,141 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr options.onRequestRecord?.(record); const controller = new AbortController(); - const forwardTimer = setTimeout(() => { - controller.abort(new Error("Duplex broker forward budget exceeded.")); - }, budgets.forwardTimeoutMs); - const responseTimer = setTimeout(() => { - // Response-budget backstop. The forward rejection normally answers first, - // well before this deadline. This backstop answers a request whose forward - // rejection handling itself stalls, so the request never strands. One stall - // path is a response whose headers arrive but whose body reader stays - // pending through the budget, so the forward promise never settles. - if (isSafeBridgeMethod(frame.method)) { - // A safe method never changes host state, so a stalled body reader - // cannot leave a mutation half-applied. Keep the request retryable: - // return a 504 with the completed outcome and no indeterminate marker, - // so the gateway passes it through as a retryable status and never maps - // it to a terminal 409. - respond( - frame.id, - { - status: 504, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - retryable: true, - }), - }, - "completed", - "error", - ); - return; - } - // The method may mutate host state, and the broker cannot prove the host - // applied no mutation once the response budget passes. Return a - // non-retryable 504 and mark the outcome indeterminate, so the gateway - // maps it to a terminal 409 and no caller double-applies the mutation. - respond( - frame.id, - { - status: 504, - headers: { - "content-type": "application/json", - "x-paperclip-bridge-outcome": "indeterminate", - }, - body: JSON.stringify({ - error: "Duplex broker response budget exceeded.", - outcome: "indeterminate", - retryable: false, - }), - }, - "indeterminate", - "error", - ); - }, budgets.responseBudgetMs); - forwardTimer.unref?.(); + // Start the response-budget backstop at the request envelope, so it bounds the + // whole request: the body reassembly plus the forward. The forward-budget timer + // starts later, when the broker starts the forward, so it bounds the forward + // call alone. + const responseTimer = setTimeout(() => respondBackstop(frame.id), budgets.responseBudgetMs); responseTimer.unref?.(); - pending.set(frame.id, { + const entry: PendingRequest = { controller, responded: false, - forwardTimer, + forwardTimer: null, responseTimer, dispatchStartMs: record.dispatchStartMs, - }); + method: frame.method, + reassembled: null, + forwardSettled: false, + releaseForwardTokens, + }; + pending.set(frame.id, entry); + // Route the following `body_chunk` frames for this id to the reassembler. + reassembling.add(frame.id); + // Reassemble the request body from the `body_chunk` frames, then forward it. + // The reassembler chooses the memory path or the spill path from the envelope + // `bodyByteCount`, so the broker never holds a whole large request body in + // memory. A reassembly error is a terminal protocol failure. + receiver.begin(frame.id, frame.bodyByteCount).then( + (body) => onBodyReady(frame, entry, body), + (error) => onBodyError(frame.id, entry, error), + ); + }; - forwardRequest(frame, { signal: controller.signal }).then( - (result) => { - // Keep the outcome classification consistent with the file path. A - // possibly-committed mutation carries the indeterminate marker header, so - // map it to the indeterminate outcome. Any other result is completed. - const outcome: DuplexResponseOutcome = - result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate" - ? "indeterminate" - : "completed"; - // The host delivered a real response, so the request span outcome is `ok`. - // A host application status (200, a 4xx, a 5xx) is still a delivered - // response; only a broker-synthesized failure below is `error`. - respond(frame.id, result, outcome, "ok"); + const respondBackstop = (id: string): void => { + // Response-budget backstop. The forward rejection normally answers first, well + // before this deadline. This backstop answers a request whose body never + // completes its reassembly, or whose forward rejection handling itself stalls, + // so the request never strands. One stall path is a response whose headers + // arrive but whose body reader stays pending through the budget, so the forward + // promise never settles. + const entry = pending.get(id); + if (!entry || entry.responded) return; + if (isSafeBridgeMethod(entry.method)) { + // A safe method never changes host state, so a stalled body reader cannot + // leave a mutation half-applied. Keep the request retryable: return a 504 + // with the completed outcome and no indeterminate marker, so the gateway + // passes it through as a retryable status and never maps it to a terminal 409. + respond( + id, + { + status: 504, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + error: "Duplex broker response budget exceeded.", + retryable: true, + }), + }, + "completed", + "error", + ); + return; + } + // The method may mutate host state, and the broker cannot prove the host + // applied no mutation once the response budget passes. Return a non-retryable + // 504 and mark the outcome indeterminate, so the gateway maps it to a terminal + // 409 and no caller double-applies the mutation. + respond( + id, + { + status: 504, + headers: { + "content-type": "application/json", + "x-paperclip-bridge-outcome": "indeterminate", + }, + body: JSON.stringify({ + error: "Duplex broker response budget exceeded.", + outcome: "indeterminate", + retryable: false, + }), }, - (error) => { - if (controller.signal.aborted) { + "indeterminate", + "error", + ); + }; + + // The request body reassembled. Start the forward now, so the forward budget + // bounds the forward call alone. The broker passes the reassembled body to the + // forward handler, so the handler streams it to the host API path. + const onBodyReady = ( + frame: DuplexRequestFrame, + entry: PendingRequest, + body: ReassembledBody, + ): void => { + reassembling.delete(frame.id); + if (stopped || state !== "open" || entry.responded) { + // The channel died, or the response backstop already answered, so no forward + // runs. Dispose the body and release the request tokens, so no spill file, + // arena reservation, or ledger token lingers, and drop the request from the + // pending and orphan maps. + void body.dispose(); + pending.delete(frame.id); + orphanedForwards.delete(frame.id); + entry.releaseForwardTokens(); + return; + } + entry.reassembled = body; + const forwardTimer = setTimeout(() => { + entry.controller.abort(new Error("Duplex broker forward budget exceeded.")); + }, budgets.forwardTimeoutMs); + forwardTimer.unref?.(); + entry.forwardTimer = forwardTimer; + + // The forward promise has one cleanup owner. Both settle handlers mark the + // forward settled and answer the gateway. The `finally` then releases the + // request tokens exactly one time, disposes the reassembled body, and removes + // the request from the pending map and the orphan map. + const markForwardSettled = (): void => { + entry.forwardSettled = true; + }; + forwardRequest(frame, { signal: entry.controller.signal, body }) + .then( + (result) => { + markForwardSettled(); + // Keep the outcome classification consistent with the file path. A + // possibly-committed mutation carries the indeterminate marker header, so + // map it to the indeterminate outcome. Any other result is completed. + const outcome: DuplexResponseOutcome = + result.headers?.["x-paperclip-bridge-outcome"] === "indeterminate" + ? "indeterminate" + : "completed"; + // The host delivered a real response, so the request span outcome is `ok`. + // A host application status (200, a 4xx, a 5xx) is still a delivered + // response; only a broker-synthesized failure below is `error`. + respond(frame.id, result, outcome, "ok"); + }, + (error) => { + markForwardSettled(); + if (entry.controller.signal.aborted) { // The forward budget aborted the call. A safe method never changes // host state, so a forward timeout stays retryable for it. Return a // 504 with the completed outcome and no indeterminate marker, so the @@ -771,8 +1252,61 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr "indeterminate", "error", ); - }, - ); + }, + ) + .finally(() => { + // The single forward cleanup owner. Release the request tokens, dispose the + // reassembled body, and drop the request from the pending and orphan maps, + // each exactly one time. + pending.delete(frame.id); + orphanedForwards.delete(frame.id); + entry.releaseForwardTokens(); + if (entry.reassembled !== null) { + void entry.reassembled.dispose(); + entry.reassembled = null; + } + }); + }; + + // The request body reassembly failed. A malformed chunk, a size mismatch, an + // overrun, or a truncation is a terminal protocol failure, so the broker fails + // the channel closed. A `channel_closed` error is the broker's own teardown of an + // in-flight body, so it is not a fresh loss. + const onBodyError = (id: string, entry: PendingRequest, error: unknown): void => { + reassembling.delete(id); + // The forward never ran, so release its request tokens and drop the request + // from the pending and orphan maps. The release is idempotent, so a teardown + // that already released the tokens keeps the count correct. + pending.delete(id); + orphanedForwards.delete(id); + entry.releaseForwardTokens(); + if (stopped) return; + if (error instanceof DuplexBodyError && error.code === "channel_closed") return; + recordLoss("protocol_failure", errorMessage(error)); + }; + + // Route one `body_chunk` frame. A chunk for an id under reassembly goes to the + // reassembler, which validates it and appends it. A chunk for a refused or + // duplicate id drains against its recorded byte count and drops. A chunk with no + // matching envelope is a terminal protocol failure. + const handleBodyChunk = (frame: DuplexBodyChunkFrame): void => { + if (reassembling.has(frame.id)) { + const result = receiver.pushChunk(frame); + if (!result.ok) recordLoss("protocol_failure", result.error.message); + return; + } + const remaining = draining.get(frame.id); + if (remaining !== undefined) { + // Drain and drop a chunk of a refused body. The broker does not validate a + // drained chunk; it only tracks the byte progress, so it stops draining once + // the refused body ends. A further chunk after that is a chunk with no + // envelope. + const next = remaining - Buffer.byteLength(frame.data, "base64"); + if (next > 0) draining.set(frame.id, next); + else draining.delete(frame.id); + return; + } + recordLoss("protocol_failure", "body_chunk arrived with no matching envelope"); }; const handleFrame = (frame: DuplexFrame): void => { @@ -780,6 +1314,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr case "request": dispatch(frame); return; + case "body_chunk": + handleBodyChunk(frame); + return; case "close": // The gateway asked for an orderly close. The gateway sends this frame on // the agent's orderly completion, so mark the completion on the ordered @@ -813,7 +1350,14 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr } }; - const onExit = (): void => { + const onExit = (exit: { exitCode: number | null; transportClosed?: boolean }): void => { + // A reason-less transport close is not a process exit. Record it as a distinct + // loss, so a transport close stays legible in the loss taxonomy. A real process + // exit stays `channel_exit` -> `provider_exit`. + if (exit.transportClosed === true) { + recordLoss("transport_closed", "The sandbox channel transport closed with no exit."); + return; + } recordLoss("channel_exit", "The sandbox channel process exited."); }; @@ -838,6 +1382,8 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr setState("closing"); clearHeartbeat(); clearPending(); + releaseSeenRequestIdTokens(); + decoder.dispose(); // Send an orderly close frame. Ignore a write failure here; the broker is // already closing, so a dead channel needs no loss record. try { @@ -861,6 +1407,9 @@ export function createDuplexBridgeBroker(options: DuplexBrokerOptions): DuplexBr if (closeTimer !== undefined) clearTimeout(closeTimer); recordLoss("close_timeout", errorMessage(error)); } + // Remove the spill directory and every in-flight body. The broker owns the + // reassembler, so the orderly close cleans it up. + await finalizeReceiver(); })(); return closePromise; }; diff --git a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts index a4e2942c974..91485d79080 100644 --- a/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts +++ b/packages/adapter-utils/src/duplex-bridge-local-e2e.test.ts @@ -4,6 +4,7 @@ import net from "node:net"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { Readable } from "node:stream"; import { StringDecoder } from "node:string_decoder"; import { afterEach, describe, expect, it } from "vitest"; @@ -14,6 +15,9 @@ import { } from "./sandbox-callback-bridge.js"; import { DuplexFrameDecoder, + DUPLEX_BODY_CHUNK_RAW_BYTES, + DUPLEX_FRAME_VERSION, + encodeDuplexFrame, type DuplexFrame, type DuplexReadyFrame, type DuplexRequestFrame, @@ -23,6 +27,7 @@ import { type DuplexBridgeBroker, type DuplexBrokerForwardResult, } from "./duplex-bridge-broker.js"; +import type { ReassembledBody } from "./duplex-body-spool.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; /** @@ -299,7 +304,7 @@ async function createHarness(options: HarnessOptions = {}): Promise => { forwardedRequests.push(request); // Keep the real route allowlist on the forward seam. The broker forwards @@ -334,12 +339,16 @@ async function createHarness(options: HarnessOptions = {}): Promise; + forwardInit.duplex = "half"; + } + const response = await fetch(target, forwardInit); const body = await response.text(); const outHeaders: Record = {}; response.headers.forEach((value, key) => { @@ -349,7 +358,7 @@ async function createHarness(options: HarnessOptions = {}): Promise undefined, @@ -456,7 +465,7 @@ describe("duplex bridge local end-to-end harness", () => { const ready = harness.observedFrames.find( (frame) => frame.type === "ready", ) as DuplexReadyFrame; - expect(ready.version).toBe(1); + expect(ready.version).toBe(2); expect(ready.nonce).toBe(harness.nonce); // READY is liveness only. It carries no address data. expect((ready as unknown as Record).address).toBeUndefined(); @@ -643,4 +652,90 @@ describe("duplex bridge local end-to-end harness", () => { expect(harness.api.listening()).toBe(false); expect(harness.api.openSocketCount()).toBe(0); }, 20000); + + it("fails a request with a local 502 when the host sends a malformed response chunk", async () => { + const RAW = DUPLEX_BODY_CHUNK_RAW_BYTES; + // Each case declares one malformed body_chunk a broken or hostile host could + // send. The gateway must reject it at once with a local 502. It must not grow + // its response reassembly buffer until the response timeout. `bodyByteCount` + // sets the declared body size on the response envelope. `data` is the base64 + // payload of the one injected chunk. + const cases: Array<{ name: string; bodyByteCount: number; data: string; error: string }> = [ + { name: "an empty chunk", bodyByteCount: RAW, data: "", error: "duplex response body_chunk is empty" }, + { + name: "an undersized non-final chunk", + bodyByteCount: RAW + 8, + data: Buffer.alloc(4, 1).toString("base64"), + error: "duplex response body_chunk has the wrong size", + }, + { + name: "an oversized chunk", + bodyByteCount: RAW + 8, + data: Buffer.alloc(RAW + 4, 1).toString("base64"), + error: "duplex response body_chunk has the wrong size", + }, + { + name: "an overrun past the declared size", + bodyByteCount: 10, + data: Buffer.alloc(200, 1).toString("base64"), + error: "duplex response body overruns the declared size", + }, + { + name: "a non-canonical base64 chunk", + bodyByteCount: 10, + data: "AB==", + error: "duplex response body_chunk is not canonical base64", + }, + ]; + + const harness = await createHarness(); + harnesses.push(harness); + await harness.waitFor(readyPredicate(harness), "the gateway to become ready"); + + // Hold the broker forward open, so only the injected frames answer each + // request. The gateway reassembles the response body itself, so a raw + // malformed chunk on its stdin drives the reject path under test. + harness.setForwardMode("hang"); + + for (const testCase of cases) { + const forwardedBefore = harness.forwardedRequests.length; + const pendingFetch = fetch(`${harness.baseUrl}/api/agents/me`, { + headers: { authorization: `Bearer ${harness.bridgeToken}` }, + }); + void pendingFetch.catch(() => undefined); + await harness.waitFor( + () => harness.forwardedRequests.length > forwardedBefore, + `the broker to receive the forwarded request for ${testCase.name}`, + ); + const id = harness.forwardedRequests[harness.forwardedRequests.length - 1].id; + + // Inject the response envelope, then the one malformed body_chunk, straight + // to the gateway stdin. This bypasses the broker response encoder, so the + // gateway sees the exact bytes a broken or hostile host could send. + harness.child.stdin.write( + encodeDuplexFrame({ + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: 200, + headers: {}, + bodyByteCount: testCase.bodyByteCount, + outcome: "completed", + }), + ); + harness.child.stdin.write( + encodeDuplexFrame({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id, + seq: 0, + data: testCase.data, + }), + ); + + const response = await pendingFetch; + expect(response.status).toBe(502); + await expect(response.json()).resolves.toEqual({ error: testCase.error }); + } + }, 20000); }); diff --git a/packages/adapter-utils/src/duplex-frame-codec.test.ts b/packages/adapter-utils/src/duplex-frame-codec.test.ts index e9c00b0b75d..0fddd5e3cc2 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.test.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.test.ts @@ -8,9 +8,11 @@ import { DuplexFrameDecoder, decodeDuplexLine, encodeDuplexFrame, + encodeDuplexFrameChecked, type DuplexDecodeResult, type DuplexFrame, } from "./duplex-frame-codec.js"; +import { DuplexAggregateByteLedger } from "./duplex-aggregate-byte-ledger.js"; // One expected decode result in the fixture: either a decoded frame or the code // of a protocol error. The codec must never throw on the read path. @@ -33,10 +35,20 @@ interface Vector { expected: ExpectedResult[]; } +// One encode-bound vector: a frame, the size limit, and the expected checked +// encode result. An ok result must also decode back to the same frame. +interface EncodeVector { + name: string; + maxFrameBytes: number; + frame: DuplexFrame; + expected: { ok: true } | { ok: false; error: string }; +} + interface Fixture { frameVersion: number; defaultMaxFrameBytes: number; vectors: Vector[]; + encodeVectors: EncodeVector[]; } const fixturePath = fileURLToPath( @@ -114,7 +126,7 @@ describe("round-trip", () => { ), ), ); - for (const type of ["request", "response", "ready", "heartbeat", "close", "error"]) { + for (const type of ["request", "response", "body_chunk", "ready", "heartbeat", "close", "error"]) { expect(types).toContain(type); } }); @@ -200,8 +212,8 @@ describe("streaming decoder behavior", () => { method: "POST", path: "/x", query: "", - headers: {}, - body: "😀", + headers: { "x-note": "😀" }, + bodyByteCount: 0, }; const bytes = Buffer.from(encodeDuplexFrame(frame), "utf8"); // Split inside the four-byte emoji, right before a continuation byte. @@ -263,7 +275,7 @@ describe("request id byte bound", () => { path: "/", query: "", headers: {}, - body: "", + bodyByteCount: 0, }); } @@ -302,7 +314,7 @@ describe("request id byte bound", () => { id, status: 200, headers: {}, - body: "", + bodyByteCount: 0, outcome: "completed", }), ); @@ -310,3 +322,186 @@ describe("request id byte bound", () => { if (!result.ok) expect(result.error.code).toBe("id_too_large"); }); }); + +describe("decoder aggregate byte ledger charging", () => { + // The host injects the process-owned ledger into the decoder. The decoder + // charges the raw partial-frame bytes it retains between chunks, plus the peak + // replacement buffer it allocates on concat. It never retains an uncharged + // buffer, and it fails closed when a reservation would pass the ceiling. + function makeLedger(ceilingBytes = 1024): DuplexAggregateByteLedger { + return new DuplexAggregateByteLedger({ ceilingBytes }); + } + + it("charges the retained partial frame, then releases it when the frame completes", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + const line = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const bytes = Buffer.from(line, "utf8"); + + // A partial frame with no newline stays retained and stays charged. + const first = decoder.push(bytes.subarray(0, 3)); + expect(first).toHaveLength(0); + expect(ledger.bytesInUse).toBe(3); + expect(ledger.liveTokenCount).toBe(1); + + // The completing chunk delivers the frame and releases the retained bytes. + const second = decoder.push(bytes.subarray(3)); + expect(second).toHaveLength(1); + expect(second[0].ok).toBe(true); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("covers the peak concat buffer and settles on the remaining bytes", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + const first = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const partial = '{"version":1,"type":"hea'; + // One full frame plus a partial second frame arrive in one chunk. The full + // frame decodes and releases; the partial tail stays charged at its exact size. + const results = decoder.push(Buffer.from(first + partial, "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(true); + expect(ledger.bytesInUse).toBe(Buffer.byteLength(partial, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + }); + + it("fails closed when a chunk would pass the ceiling and retains nothing", () => { + const ledger = makeLedger(16); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + const results = decoder.push(Buffer.from("z".repeat(64), "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(false); + if (!results[0].ok) { + expect(results[0].error.code).toBe("aggregate_bytes_exceeded"); + } + // The decoder retained nothing, so the ledger stays at zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("resynchronizes on the next newline after an aggregate rejection", () => { + const ledger = makeLedger(64); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + // A first chunk over the ceiling fails closed. + const rejected = decoder.push(Buffer.from("z".repeat(128), "utf8")); + expect(rejected[0].ok).toBe(false); + + // The next chunk starts with a newline, so the decoder drops the stale tail + // and decodes the good frame that follows. + const good = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const results = decoder.push(Buffer.from(`\n${good}`, "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(true); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("releases the retained token when an oversized frame is discarded", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ maxFrameBytes: 16, aggregateByteLedger: ledger }); + // An oversized frame with no newline is discarded and its bytes are released. + const results = decoder.push(Buffer.from("z".repeat(64), "utf8")); + expect(results).toHaveLength(1); + expect(results[0].ok).toBe(false); + if (!results[0].ok) expect(results[0].error.code).toBe("frame_too_large"); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + }); + + it("releases the retained partial frame on dispose", () => { + const ledger = makeLedger(); + const decoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + decoder.push(Buffer.from('{"version":1', "utf8")); + expect(ledger.bytesInUse).toBeGreaterThan(0); + decoder.dispose(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + // A second dispose is a no-op and records no accounting defect. + decoder.dispose(); + expect(ledger.bytesInUse).toBe(0); + }); + + it("charges nothing when the caller injects no ledger", () => { + const decoder = new DuplexFrameDecoder(); + const results = decoder.push(Buffer.from('{"version":1', "utf8")); + expect(results).toHaveLength(0); + // No ledger means no charge; the decoder still buffers and decodes as before. + const line = encodeDuplexFrame({ version: DUPLEX_FRAME_VERSION, type: "heartbeat" }); + const done = decoder.push(Buffer.from(`}\n${line}`, "utf8")); + // The stitched first frame is malformed JSON; the second is a valid heartbeat. + expect(done.some((result) => result.ok)).toBe(true); + }); +}); + +describe("size-checked encode", () => { + it("runs every shared encode vector and matches the expected result", () => { + // Every codec copy runs the same encode vectors. This copy proves it enforces + // the same bound the embedded gateway copy enforces. + for (const vector of fixture.encodeVectors) { + const result = encodeDuplexFrameChecked(vector.frame, vector.maxFrameBytes); + expect(result.ok).toBe(vector.expected.ok); + if (result.ok) { + // An ok line ends with one newline and decodes back to the same frame, so + // the encode guard never truncates or passes a bad frame through. + expect(result.line.endsWith("\n")).toBe(true); + expect(result.line.slice(0, -1)).not.toContain("\n"); + const decoded = decodeDuplexLine(result.line.slice(0, -1)); + expect(decoded.ok).toBe(true); + if (decoded.ok) expect(decoded.frame).toEqual(vector.frame); + } else if (!vector.expected.ok) { + expect(result.error.code).toBe(vector.expected.error); + } + } + }); + + it("rejects an over-bound frame with a typed outcome and never throws", () => { + const frame: DuplexFrame = { + version: DUPLEX_FRAME_VERSION, + type: "response", + id: "big", + status: 200, + headers: { "x-pad": "x".repeat(2_000) }, + bodyByteCount: 0, + outcome: "completed", + }; + expect(() => encodeDuplexFrameChecked(frame, 1_000)).not.toThrow(); + const result = encodeDuplexFrameChecked(frame, 1_000); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.code).toBe("frame_too_large"); + }); + + it("measures the bound in bytes without the trailing newline, so the boundary is inclusive", () => { + // Build a frame whose encoded JSON is exactly N bytes. The guard measures the + // JSON without the newline, so N bytes encodes and N+1 bytes fails. This is the + // same boundary the decoder applies, so encode and decode agree exactly. + // Pad a header value to grow the encoded JSON by an exact byte count. The + // "x-pad" value adds one byte per character, so the frame lands on an exact + // size the guard measures without the trailing newline. + const pad = (n: number): DuplexFrame => ({ + version: DUPLEX_FRAME_VERSION, + type: "response", + id: "boundary", + status: 200, + headers: { "x-pad": "x".repeat(n) }, + bodyByteCount: 0, + outcome: "completed", + }); + const baseBytes = Buffer.byteLength(JSON.stringify(pad(0)), "utf8"); + const limit = baseBytes + 10; + const atLimit = pad(10); + expect(Buffer.byteLength(JSON.stringify(atLimit), "utf8")).toBe(limit); + const okResult = encodeDuplexFrameChecked(atLimit, limit); + expect(okResult.ok).toBe(true); + const overResult = encodeDuplexFrameChecked(pad(11), limit); + expect(overResult.ok).toBe(false); + if (!overResult.ok) expect(overResult.error.code).toBe("frame_too_large"); + }); + + it("defaults the bound to the default max frame bytes", () => { + const frame: DuplexFrame = { version: DUPLEX_FRAME_VERSION, type: "heartbeat" }; + const result = encodeDuplexFrameChecked(frame); + expect(result.ok).toBe(true); + if (result.ok) expect(result.line).toBe(encodeDuplexFrame(frame)); + }); +}); diff --git a/packages/adapter-utils/src/duplex-frame-codec.ts b/packages/adapter-utils/src/duplex-frame-codec.ts index 2efdb384a23..1013dbf106c 100644 --- a/packages/adapter-utils/src/duplex-frame-codec.ts +++ b/packages/adapter-utils/src/duplex-frame-codec.ts @@ -17,8 +17,22 @@ * keeps one bad frame from crashing the read loop. */ +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + type DuplexAggregateByteLedger, + type ReservationToken, +} from "./duplex-aggregate-byte-ledger.js"; + /** The wire version this codec reads and writes. */ -export const DUPLEX_FRAME_VERSION = 1; +export const DUPLEX_FRAME_VERSION = 2; + +/** + * The fixed raw byte size of one body slice. The sender splits a body into + * `body_chunk` frames of this raw size, except the final chunk, which holds the + * remaining bytes. The base64 text of one full slice is about 349 KiB, so each + * encoded `body_chunk` frame stays well under {@link DEFAULT_MAX_DUPLEX_FRAME_BYTES}. + */ +export const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024; /** * The default maximum size of one frame, in bytes. The decoder rejects a longer @@ -48,6 +62,7 @@ const EMPTY = Buffer.alloc(0); export const DUPLEX_FRAME_TYPES = { request: "request", response: "response", + body_chunk: "body_chunk", ready: "ready", heartbeat: "heartbeat", close: "close", @@ -57,7 +72,12 @@ export const DUPLEX_FRAME_TYPES = { /** The outcome of a response frame. A loss response carries a non-completed outcome. */ export type DuplexResponseOutcome = "completed" | "indeterminate" | "unavailable"; -/** A request frame. The gateway forwards it to the host API path. */ +/** + * A request envelope frame. The gateway forwards it to the host API path. The + * envelope carries `bodyByteCount`, the raw byte length of the request body. The + * body itself rides one or more `body_chunk` frames that share the same `id`. A + * `bodyByteCount` of 0 means an empty body and no `body_chunk` frame follows. + */ export interface DuplexRequestFrame { version: number; type: "request"; @@ -66,20 +86,43 @@ export interface DuplexRequestFrame { path: string; query: string; headers: Record; - body: string; + bodyByteCount: number; } -/** A response frame. The host returns it for one request id. */ +/** + * A response envelope frame. The host returns it for one request id. The + * envelope carries `bodyByteCount`, the raw byte length of the response body. + * The body itself rides one or more `body_chunk` frames that share the same + * `id`. A `bodyByteCount` of 0 means an empty body and no `body_chunk` frame + * follows. + */ export interface DuplexResponseFrame { version: number; type: "response"; id: string; status: number; headers: Record; - body: string; + bodyByteCount: number; outcome: DuplexResponseOutcome; } +/** + * A body-chunk frame. One frame carries one raw body slice as base64 text in + * `data`. The frames for one body share the `id` of the envelope. `seq` starts + * at 0 and increases by one for each next chunk, with no gap and no repeat. Each + * raw slice is {@link DUPLEX_BODY_CHUNK_RAW_BYTES} bytes, except the final slice, + * which holds the remaining bytes. The receiver reassembles the slices in `seq` + * order and stops when the total raw byte count equals the envelope + * `bodyByteCount`. + */ +export interface DuplexBodyChunkFrame { + version: number; + type: "body_chunk"; + id: string; + seq: number; + data: string; +} + /** * The READY control frame. The gateway sends it one time after it binds the * host-assigned listener port. READY is a liveness signal, not an address @@ -123,6 +166,7 @@ export interface DuplexErrorFrame { export type DuplexFrame = | DuplexRequestFrame | DuplexResponseFrame + | DuplexBodyChunkFrame | DuplexReadyFrame | DuplexHeartbeatFrame | DuplexCloseFrame @@ -134,7 +178,8 @@ export type DuplexProtocolErrorCode = | "unknown_type" | "version_mismatch" | "frame_too_large" - | "id_too_large"; + | "id_too_large" + | "aggregate_bytes_exceeded"; /** A decode-time protocol error. The read path returns it; it never throws. */ export interface DuplexProtocolError { @@ -147,6 +192,15 @@ export type DuplexDecodeResult = | { ok: true; frame: DuplexFrame } | { ok: false; error: DuplexProtocolError }; +/** + * One size-checked encode result: one line, or a `frame_too_large` error. The + * shape mirrors {@link DuplexDecodeResult}, so the encode side reports the + * over-limit case as a typed outcome, not an exception. + */ +export type DuplexEncodeResult = + | { ok: true; line: string } + | { ok: false; error: { code: "frame_too_large"; message: string } }; + const RESPONSE_OUTCOMES: ReadonlySet = new Set([ "completed", "indeterminate", @@ -178,6 +232,11 @@ function idWithinLimit(id: string): boolean { return Buffer.byteLength(id, "utf8") <= DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES; } +/** Return true when the value is a safe integer that is zero or positive. */ +function isSafeNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + /** * Encode one frame to a single line of JSON with a trailing newline. `JSON.stringify` * escapes any newline inside a string value, so the returned line holds no @@ -187,6 +246,30 @@ export function encodeDuplexFrame(frame: DuplexFrame): string { return `${JSON.stringify(frame)}\n`; } +/** + * Encode one frame to a single line and enforce the maximum frame size. The + * function measures the encoded JSON in bytes, without the trailing newline, so + * it matches the decoder bound exactly: a line the decoder accepts encodes, and a + * line the decoder rejects returns a `frame_too_large` result. The function never + * throws; it reports the over-limit case as a typed outcome. + * + * The size bound follows request and response bodies the host does not control, + * so an over-limit frame is an expected condition, not a programming error. Every + * write path that can carry a large body uses this function, so no path emits a + * frame the peer decoder rejects. The bound applies to every frame type; the + * guard is a no-op for a small control frame. + */ +export function encodeDuplexFrameChecked( + frame: DuplexFrame, + maxFrameBytes: number = DEFAULT_MAX_DUPLEX_FRAME_BYTES, +): DuplexEncodeResult { + const json = JSON.stringify(frame); + if (Buffer.byteLength(json, "utf8") > maxFrameBytes) { + return { ok: false, error: { code: "frame_too_large", message: "frame exceeds the maximum size" } }; + } + return { ok: true, line: `${json}\n` }; +} + /** * Decode one line (no trailing newline) to a frame or a protocol error. The * streaming decoder calls this for each complete line. It is exported so a @@ -221,6 +304,8 @@ function validateFrame(frame: Record): DuplexDecodeResult { return validateRequest(frame); case "response": return validateResponse(frame); + case "body_chunk": + return validateBodyChunk(frame); case "ready": return validateReady(frame); case "heartbeat": @@ -240,7 +325,7 @@ function validateRequest(frame: Record): DuplexDecodeResult { typeof frame.method !== "string" || typeof frame.path !== "string" || typeof frame.query !== "string" || - typeof frame.body !== "string" || + !isSafeNonNegativeInteger(frame.bodyByteCount) || !isStringRecord(frame.headers) ) { return fail("malformed_frame", "request frame has a missing or wrong-typed field"); @@ -258,7 +343,7 @@ function validateResponse(frame: Record): DuplexDecodeResult { if ( typeof frame.id !== "string" || typeof frame.status !== "number" || - typeof frame.body !== "string" || + !isSafeNonNegativeInteger(frame.bodyByteCount) || !isStringRecord(frame.headers) || typeof frame.outcome !== "string" || !RESPONSE_OUTCOMES.has(frame.outcome) @@ -273,6 +358,29 @@ function validateResponse(frame: Record): DuplexDecodeResult { return ok(frame as unknown as DuplexResponseFrame); } +/** + * Validate a `body_chunk` frame at the structural level. The codec is stateless, + * so it checks only the per-frame shape here: + * - `id` is a string within {@link DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES}. + * - `seq` is a safe integer that is zero or positive. + * - `data` is a string. + * The receive-side reassembler owns the stateful checks: the base64 canonical + * form, the fixed slice size, the `seq` order, and the total against + * `bodyByteCount`. The codec never decodes `data` here. + */ +function validateBodyChunk(frame: Record): DuplexDecodeResult { + if (typeof frame.id !== "string" || typeof frame.data !== "string") { + return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed field"); + } + if (!isSafeNonNegativeInteger(frame.seq)) { + return fail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq"); + } + if (!idWithinLimit(frame.id)) { + return fail("id_too_large", "body_chunk frame id exceeds the maximum size"); + } + return ok(frame as unknown as DuplexBodyChunkFrame); +} + function validateReady(frame: Record): DuplexDecodeResult { // READY carries a liveness nonce, not an address. The schema is strict: a valid // READY frame holds exactly `version`, `type`, and `nonce`. The decoder rejects @@ -311,6 +419,18 @@ function validateError(frame: Record): DuplexDecodeResult { export interface DuplexFrameDecoderOptions { /** The maximum size of one frame, in bytes. Defaults to {@link DEFAULT_MAX_DUPLEX_FRAME_BYTES}. */ maxFrameBytes?: number; + /** + * The process-owned aggregate byte ledger. When present, the decoder reserves + * the exact bytes of the raw partial frame it retains between chunks, and the + * peak replacement buffer it allocates on concat, against this ledger under the + * `decoder_buffer` owner. The host injects the same ledger object it injects at + * every other retention site, so one gauge bounds the aggregate retained bytes. + * When absent the decoder charges nothing and behaves as before. + * + * The generated sandbox decoder runs in a separate operating-system process. It + * cannot share this host ledger. It receives its own separate cap instead. + */ + aggregateByteLedger?: DuplexAggregateByteLedger; } /** @@ -331,15 +451,41 @@ export class DuplexFrameDecoder { private buffer: Buffer = EMPTY; private discarding = false; private readonly maxFrameBytes: number; + private readonly ledger: DuplexAggregateByteLedger | null; + /** + * The token for the bytes currently retained in `this.buffer`. Its byte amount + * equals `this.buffer.length` at the end of every `push`. It is `null` when the + * buffer is empty or the decoder has no ledger. + */ + private retainedToken: ReservationToken | null = null; constructor(options: DuplexFrameDecoderOptions = {}) { this.maxFrameBytes = options.maxFrameBytes ?? DEFAULT_MAX_DUPLEX_FRAME_BYTES; + this.ledger = options.aggregateByteLedger ?? null; } /** Feed one chunk. Return the frames and protocol errors that complete on it. */ push(chunk: Buffer | Uint8Array | string): DuplexDecodeResult[] { const incoming = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk); + + // Reserve the incoming bytes before the concat allocates the replacement + // buffer. The retained token already covers the current buffer, so the two + // tokens together cover the peak `old + incoming` allocation. A rejected + // reservation fails closed: the decoder drops the incoming chunk, releases + // the retained buffer, resynchronizes at the next newline, and reports the + // fixed aggregate rejection marker. It retains nothing uncharged. + let incomingToken: ReservationToken | null = null; + if (this.ledger && incoming.length > 0) { + incomingToken = this.ledger.reserve("decoder_buffer", incoming.length); + if (incomingToken === null) { + this.releaseRetained(); + this.buffer = EMPTY; + this.discarding = true; + return [fail("aggregate_bytes_exceeded", DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED)]; + } + } + this.buffer = this.buffer.length === 0 ? incoming : Buffer.concat([this.buffer, incoming]); @@ -378,6 +524,54 @@ export class DuplexFrameDecoder { } results.push(decodeDuplexLine(line)); } + + // Reconcile the ledger to the bytes still retained in `this.buffer`. Release + // the old retained token and the incoming token, then reserve one token for + // the remainder. The remainder is a subset of the just-released + // `old + incoming` bytes, so this reserve always fits. + this.reconcileRetained(incomingToken); return results; } + + /** + * Release the retained-buffer token and drop the buffer. The host calls this at + * channel teardown, so the decoder never leaks a `decoder_buffer` token after + * the channel ends. A second call is a no-op, because the field is already + * `null`. + */ + dispose(): void { + this.releaseRetained(); + this.buffer = EMPTY; + this.discarding = false; + } + + /** Release the retained-buffer token one time and clear the field. */ + private releaseRetained(): void { + if (this.ledger && this.retainedToken) { + this.ledger.release(this.retainedToken); + } + this.retainedToken = null; + } + + /** + * Move the ledger charge to the bytes still in `this.buffer`. Release the old + * retained token and the `incoming` token, then reserve one token for the + * remaining bytes. A `null` reserve here is an accounting defect, because the + * remainder never passes the just-released bytes; the decoder fails closed and + * drops the buffer so it retains nothing uncharged. + */ + private reconcileRetained(incomingToken: ReservationToken | null): void { + if (!this.ledger) return; + this.releaseRetained(); + if (incomingToken) { + this.ledger.release(incomingToken); + } + if (this.buffer.length > 0) { + this.retainedToken = this.ledger.reserve("decoder_buffer", this.buffer.length); + if (this.retainedToken === null) { + this.buffer = EMPTY; + this.discarding = true; + } + } + } } diff --git a/packages/adapter-utils/src/duplex-frame-vectors.json b/packages/adapter-utils/src/duplex-frame-vectors.json index 74ded507fad..62d43d1c15f 100644 --- a/packages/adapter-utils/src/duplex-frame-vectors.json +++ b/packages/adapter-utils/src/duplex-frame-vectors.json @@ -1,17 +1,17 @@ { - "frameVersion": 1, + "frameVersion": 2, "defaultMaxFrameBytes": 1000000, "description": "Shared wire-compatibility vectors for the duplex frame codec. Every codec copy decodes the same bytes. bytes is a UTF-8 byte stream; splitByteOffsets cuts it into chunks; expected lists the ordered decode results (frame or protocol-error code).", "vectors": [ { "name": "valid-request", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -21,7 +21,30 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 + } + } + ] + }, + { + "name": "valid-request-with-body-count", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-body\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":1048576}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "request", + "id": "req-body", + "method": "GET", + "path": "/api/issues/PAP-1", + "query": "expand=comments", + "headers": { + "accept": "application/json", + "content-type": "application/json" + }, + "bodyByteCount": 1048576 } } ] @@ -29,19 +52,19 @@ { "name": "valid-response-completed", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-1", "status": 200, "headers": { "content-type": "application/json" }, - "body": "{\"ok\":true}", + "bodyByteCount": 11, "outcome": "completed" } } @@ -50,19 +73,19 @@ { "name": "valid-response-indeterminate", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"outcome_indeterminate\\\"}\",\"outcome\":\"indeterminate\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-2\",\"status\":409,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":33,\"outcome\":\"indeterminate\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-2", "status": 409, "headers": { "content-type": "application/json" }, - "body": "{\"error\":\"outcome_indeterminate\"}", + "bodyByteCount": 33, "outcome": "indeterminate" } } @@ -71,43 +94,121 @@ { "name": "valid-response-unavailable", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"error\\\":\\\"bridge_unavailable\\\"}\",\"outcome\":\"unavailable\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-3\",\"status\":503,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":30,\"outcome\":\"unavailable\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-3", "status": 503, "headers": { "content-type": "application/json" }, - "body": "{\"error\":\"bridge_unavailable\"}", + "bodyByteCount": 30, "outcome": "unavailable" } } ] }, + { + "name": "valid-body-chunk", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":\"aGVsbG8gYm9keSBjaHVuaw==\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-1", + "seq": 0, + "data": "aGVsbG8gYm9keSBjaHVuaw==" + } + } + ] + }, + { + "name": "valid-body-chunk-final", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":7,\"data\":\"dGFpbA==\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-1", + "seq": 7, + "data": "dGFpbA==" + } + } + ] + }, { "name": "valid-ready", "category": "valid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\"}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "ready", "nonce": "9f8e7d6c5b4a39281706f5e4d3c2b1a0" } } ] }, + { + "name": "valid-heartbeat", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"heartbeat\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "heartbeat" + } + } + ] + }, + { + "name": "valid-close", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"close\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "close" + } + } + ] + }, + { + "name": "valid-error", + "category": "valid", + "bytes": "{\"version\":2,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n", + "roundTrip": true, + "expected": [ + { + "frame": { + "version": 2, + "type": "error", + "code": "malformed_frame", + "message": "peer reported a malformed frame" + } + } + ] + }, { "name": "invalid-ready-missing-nonce", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\"}\n", "expected": [ { "error": "malformed_frame" @@ -117,7 +218,7 @@ { "name": "invalid-ready-with-address", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"address\":\"http://127.0.0.1:47215\"}\n", "expected": [ { "error": "malformed_frame" @@ -127,57 +228,13 @@ { "name": "invalid-ready-with-port", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n", + "bytes": "{\"version\":2,\"type\":\"ready\",\"nonce\":\"9f8e7d6c5b4a39281706f5e4d3c2b1a0\",\"port\":47215}\n", "expected": [ { "error": "malformed_frame" } ] }, - { - "name": "valid-heartbeat", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"heartbeat\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "heartbeat" - } - } - ] - }, - { - "name": "valid-close", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"close\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "close" - } - } - ] - }, - { - "name": "valid-error", - "category": "valid", - "bytes": "{\"version\":1,\"type\":\"error\",\"code\":\"malformed_frame\",\"message\":\"peer reported a malformed frame\"}\n", - "roundTrip": true, - "expected": [ - { - "frame": { - "version": 1, - "type": "error", - "code": "malformed_frame", - "message": "peer reported a malformed frame" - } - } - ] - }, { "name": "invalid-not-json", "category": "invalid", @@ -211,7 +268,7 @@ { "name": "invalid-unknown-type", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"teleport\"}\n", + "bytes": "{\"version\":2,\"type\":\"teleport\"}\n", "expected": [ { "error": "unknown_type" @@ -221,7 +278,37 @@ { "name": "invalid-missing-id", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-string", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":\"10\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-negative", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":-1}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-request-body-count-fractional", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":1.5}\n", "expected": [ { "error": "malformed_frame" @@ -231,7 +318,47 @@ { "name": "invalid-bad-headers", "category": "invalid", - "bytes": "{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"accept\":7},\"bodyByteCount\":10,\"outcome\":\"completed\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-negative-seq", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":-1,\"data\":\"eA==\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-fractional-seq", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":2.5,\"data\":\"eA==\"}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-missing-data", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0}\n", + "expected": [ + { + "error": "malformed_frame" + } + ] + }, + { + "name": "invalid-body-chunk-nonstring-data", + "category": "invalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"req-1\",\"seq\":0,\"data\":42}\n", "expected": [ { "error": "malformed_frame" @@ -241,14 +368,14 @@ { "name": "partial-request-split", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "splitByteOffsets": [ - 98 + 102 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -258,7 +385,7 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 } } ] @@ -266,23 +393,24 @@ { "name": "partial-utf8-split", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1/comments\",\"query\":\"\",\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"text\\\":\\\"héllo 😀 wörld\\\"}\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-utf8\",\"method\":\"POST\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"content-type\":\"application/json\",\"x-note\":\"héllo 😀 wörld\"},\"bodyByteCount\":0}\n", "splitByteOffsets": [ - 177 + 172 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-utf8", "method": "POST", - "path": "/api/issues/PAP-1/comments", - "query": "", + "path": "/api/issues/PAP-1", + "query": "expand=comments", "headers": { - "content-type": "application/json" + "content-type": "application/json", + "x-note": "héllo 😀 wörld" }, - "body": "{\"text\":\"héllo 😀 wörld\"}" + "bodyByteCount": 0 } } ] @@ -290,16 +418,16 @@ { "name": "partial-two-frames", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n{\"version\":1,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"body\":\"{\\\"ok\\\":true}\",\"outcome\":\"completed\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n{\"version\":2,\"type\":\"response\",\"id\":\"req-1\",\"status\":200,\"headers\":{\"content-type\":\"application/json\"},\"bodyByteCount\":11,\"outcome\":\"completed\"}\n", "splitByteOffsets": [ 7, - 199, - 343 + 174, + 344 ], "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "req-1", "method": "GET", @@ -309,19 +437,19 @@ "accept": "application/json", "content-type": "application/json" }, - "body": "" + "bodyByteCount": 0 } }, { "frame": { - "version": 1, + "version": 2, "type": "response", "id": "req-1", "status": 200, "headers": { "content-type": "application/json" }, - "body": "{\"ok\":true}", + "bodyByteCount": 11, "outcome": "completed" } } @@ -330,14 +458,14 @@ { "name": "partial-no-trailing-newline", "category": "partial", - "bytes": "{\"version\":1,\"type\":\"heartbeat\"}", + "bytes": "{\"version\":2,\"type\":\"heartbeat\"}", "expected": [] }, { "name": "oversized-complete-line", "category": "oversized", "maxFrameBytes": 64, - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"q=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "expected": [ { "error": "frame_too_large" @@ -351,23 +479,33 @@ "splitByteOffsets": [ 40 ], - "bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":1,\"type\":\"heartbeat\"}\n", + "bytes": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\n{\"version\":2,\"type\":\"heartbeat\"}\n", "expected": [ { "error": "frame_too_large" }, { "frame": { - "version": 1, + "version": 2, "type": "heartbeat" } } ] }, + { + "name": "version-mismatch-lower", + "category": "versionMismatch", + "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "version_mismatch" + } + ] + }, { "name": "version-mismatch-higher", "category": "versionMismatch", - "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"body\":\"\"}\n", + "bytes": "{\"version\":3,\"type\":\"request\",\"id\":\"req-1\",\"method\":\"GET\",\"path\":\"/api/issues/PAP-1\",\"query\":\"expand=comments\",\"headers\":{\"accept\":\"application/json\",\"content-type\":\"application/json\"},\"bodyByteCount\":0}\n", "expected": [ { "error": "version_mismatch" @@ -387,19 +525,19 @@ { "name": "request-id-at-max-bytes", "category": "idBoundValid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", "roundTrip": true, "expected": [ { "frame": { - "version": 1, + "version": 2, "type": "request", "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "method": "GET", "path": "/", "query": "", "headers": {}, - "body": "" + "bodyByteCount": 0 } } ] @@ -407,12 +545,87 @@ { "name": "request-id-over-max-bytes", "category": "idBoundInvalid", - "bytes": "{\"version\":1,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"body\":\"\"}\n", + "bytes": "{\"version\":2,\"type\":\"request\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"method\":\"GET\",\"path\":\"/\",\"query\":\"\",\"headers\":{},\"bodyByteCount\":0}\n", + "expected": [ + { + "error": "id_too_large" + } + ] + }, + { + "name": "body-chunk-id-over-max-bytes", + "category": "idBoundInvalid", + "bytes": "{\"version\":2,\"type\":\"body_chunk\",\"id\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"seq\":0,\"data\":\"eA==\"}\n", "expected": [ { "error": "id_too_large" } ] } + ], + "encodeDescription": "Encode-bound vectors. Each codec copy runs the size-checked encode against the frame with the maxFrameBytes limit, then asserts the same result. An ok result must decode back to the same frame. This proves both copies enforce the same bound on encode.", + "encodeVectors": [ + { + "name": "encode-within-bound", + "maxFrameBytes": 200, + "frame": { + "version": 2, + "type": "request", + "id": "req-enc-1", + "method": "GET", + "path": "/", + "query": "", + "headers": {}, + "bodyByteCount": 5 + }, + "expected": { + "ok": true + } + }, + { + "name": "encode-body-chunk-within-bound", + "maxFrameBytes": 200, + "frame": { + "version": 2, + "type": "body_chunk", + "id": "req-enc-3", + "seq": 0, + "data": "Y2h1bmsgcGF5bG9hZA==" + }, + "expected": { + "ok": true + } + }, + { + "name": "encode-control-frame-never-rejected", + "maxFrameBytes": 200, + "frame": { + "version": 2, + "type": "heartbeat" + }, + "expected": { + "ok": true + } + }, + { + "name": "encode-over-bound", + "maxFrameBytes": 200, + "frame": { + "version": 2, + "type": "response", + "id": "req-enc-2", + "status": 200, + "headers": { + "content-type": "application/json", + "x-pad": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, + "bodyByteCount": 0, + "outcome": "completed" + }, + "expected": { + "ok": false, + "error": "frame_too_large" + } + } ] } diff --git a/packages/adapter-utils/src/duplex-telemetry.ts b/packages/adapter-utils/src/duplex-telemetry.ts index 4bc08abebf6..b31c6559183 100644 --- a/packages/adapter-utils/src/duplex-telemetry.ts +++ b/packages/adapter-utils/src/duplex-telemetry.ts @@ -35,6 +35,40 @@ export const DUPLEX_COUNTER_LOSS_TOTAL = "sandbox_duplex_loss_total"; /** The guarded counter for one leaked provider session on teardown. */ export const DUPLEX_COUNTER_SESSION_LEAK_TOTAL = "sandbox_duplex_session_leak_total"; +/** + * The process-scoped gauge for the aggregate retained bytes across every live + * duplex route. The host aggregate byte ledger sets it on each reserve and each + * release. The record carries no dynamic dimension. + */ +export const DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE = "sandbox_duplex_aggregate_bytes_in_use"; +/** + * The counter for one rejected aggregate byte reservation. The host ledger + * increments it when a reservation would pass the aggregate ceiling. The record + * carries no dynamic dimension. + */ +export const DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL = + "sandbox_duplex_aggregate_byte_reservation_rejections_total"; +/** + * The counter for one aggregate byte accounting defect. The host ledger + * increments it on a double release or a transfer of a token it does not hold. + * The record carries no dynamic dimension. + */ +export const DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL = + "sandbox_duplex_aggregate_byte_accounting_underflow_total"; + +/** + * The closed set of aggregate byte ledger metric names. A test pins this exact + * set, so a new ledger metric name needs an explicit review. Each record uses + * only closed constant dimensions and no dynamic label. The telemetry contract + * documents these metrics under "Aggregate byte ledger metrics" in + * `packages/shared/src/telemetry/README.md`. + */ +export const DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES = [ + DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, + DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, + DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, +] as const; + /** * The closed dimension-key set. Every span attribute, counter label, and event * field uses only these keys. A test asserts the exact set, so a new key never @@ -64,6 +98,9 @@ export type DuplexOutcomeValue = "ok" | "error"; * stage: the process-scoped route ceiling was full (`route_busy`), the entrypoint * sync failed (`entrypoint_sync_failed`), the broker construction failed * (`broker_construction_failed`), or the channel open failed (`channel_open_failed`). + * The `aggregate_bytes_exceeded` reason names a readiness handshake the host fell + * back because the process aggregate byte ceiling had no room for the readiness + * buffer. */ export type DuplexFallbackReason = | "gate_off" @@ -75,7 +112,8 @@ export type DuplexFallbackReason = | "ready_invalid" | "ready_nonce_mismatch" | "ready_timeout" - | "contaminated"; + | "contaminated" + | "aggregate_bytes_exceeded"; /** The class of a terminal loss, relative to the first request dispatch. */ export type DuplexLossClass = "pre_dispatch" | "post_dispatch"; @@ -85,9 +123,10 @@ export type DuplexLossClass = "pre_dispatch" | "post_dispatch"; * cause to one of these values before any sink reads it. The set covers the * loss-detection modes the transport names: a gateway stdin end of file, a * provider process exit, a heartbeat timeout, and an RPC failure. It adds - * `write_error` for a rejected host-to-sandbox write and `other` for an unknown - * cause. The host maps an unknown cause or any caught provider text to `other`, - * so no raw provider text reaches a sink. + * `write_error` for a rejected host-to-sandbox write, `transport_closed` for a + * reason-less provider transport close with no exit data, and `other` for an + * unknown cause. The host maps an unknown cause or any caught provider text to + * `other`, so no raw provider text reaches a sink. */ export const DUPLEX_LOSS_REASONS = [ "stdin_eof", @@ -95,6 +134,7 @@ export const DUPLEX_LOSS_REASONS = [ "heartbeat_timeout", "rpc_failure", "write_error", + "transport_closed", "other", ] as const; diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 0fd7cb426ed..b5c5c6b1481 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -43,18 +43,25 @@ import { type StartupTraceContext, type StartupTracer, } from "./acpx-engine/startup-timing.js"; +import { + DuplexAggregateByteLedger, + DUPLEX_AGGREGATE_TOKEN_OWNERS, + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, +} from "./duplex-aggregate-byte-ledger.js"; import { createSandboxRunLogTailFactory, type SandboxRunLogTailFactory } from "./sandbox-run-log-stream.js"; import { runChildProcess } from "./server-utils.js"; import { shellQuote } from "./ssh.js"; import type { CommandManagedDuplexChannel } from "./command-managed-runtime.js"; import { DEFAULT_MAX_DUPLEX_FRAME_BYTES, + DuplexFrameDecoder, DUPLEX_FRAME_VERSION, decodeDuplexLine, encodeDuplexFrame, type DuplexRequestFrame, type DuplexResponseFrame, } from "./duplex-frame-codec.js"; +import { splitBodyIntoChunkFrames } from "./duplex-body-spool.js"; import { assertNestedDuplexBrokerBudgets, createDuplexBridgeBroker, @@ -66,10 +73,14 @@ import { } from "./duplex-bridge-broker.js"; import { createDuplexTelemetry, + DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES, + DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL, + DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL, DUPLEX_COUNTER_CHANNEL_OPEN_TOTAL, DUPLEX_COUNTER_FALLBACK_TOTAL, DUPLEX_COUNTER_LOSS_TOTAL, DUPLEX_DIMENSION_KEYS, + DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE, DUPLEX_SPAN_CHANNEL_OPEN, DUPLEX_SPAN_REQUEST, DUPLEX_TRANSPORT_EVENT, @@ -82,6 +93,51 @@ import { const execFileAsync = promisify(execFile); +/** + * Emit one duplex request the way the sandbox gateway sends it under the chunked + * body protocol: the envelope frame that carries `bodyByteCount`, then the + * `body_chunk` frames that carry the body. A non-empty body rides one or more + * `body_chunk` frames that share the envelope id; an empty body sends no chunk. + */ +function emitDuplexRequest( + control: { emitData: (chunk: string) => void }, + frame: Omit, + bodyText: string, +): void { + const body = Buffer.from(bodyText, "utf8"); + control.emitData(encodeDuplexFrame({ ...frame, bodyByteCount: body.length })); + for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { + control.emitData(encodeDuplexFrame(chunk)); + } +} + +/** + * Send one duplex response the way the host broker sends it under the chunked + * body protocol: the envelope frame that carries `bodyByteCount`, then the + * `body_chunk` frames that carry the body. The gateway reassembles the body from + * the chunk frames before it writes the HTTP response. + */ +function sendDuplexResponse( + send: (frame: Record) => void, + response: { id: string | undefined; status: number; headers: Record; outcome?: string }, + bodyText: string, +): void { + const id = response.id ?? ""; + const body = Buffer.from(bodyText, "utf8"); + send({ + version: DUPLEX_FRAME_VERSION, + type: "response", + id, + status: response.status, + headers: response.headers, + bodyByteCount: body.length, + ...(response.outcome !== undefined ? { outcome: response.outcome } : {}), + }); + for (const chunk of splitBodyIntoChunkFrames(id, body, DUPLEX_FRAME_VERSION)) { + send(chunk as unknown as Record); + } +} + type RecordedSpan = { name: string; parentName: string | null; ended: boolean }; /** @@ -2678,6 +2734,155 @@ describe("sandbox adapter execution targets", () => { } }); + it("charges the response-body bytes against the host aggregate ledger and releases every token on success", async () => { + // The host stamps one process-owned aggregate byte ledger on the sandbox + // target. The forward response-body reader charges its retained bytes against + // that ledger. A successful read charges the chunk bytes and the + // concatenation buffer, then releases every token, so the ledger returns to + // zero after the forward completes. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-ok-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); + await mkdir(runtimeRootDir, { recursive: true }); + + const responseBody = JSON.stringify({ id: "issue-1" }); + const apiServer = createServer((_req, res) => { + res.writeHead(200, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(responseBody, "utf8")), + }); + res.end(responseBody); + }); + await new Promise((resolve, reject) => { + apiServer.once("error", reject); + apiServer.listen(0, "127.0.0.1", () => resolve()); + }); + const address = apiServer.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the bridge test API server to listen on a TCP port."); + } + + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd, + runner: createLocalSandboxRunner(), + timeoutMs: 30_000, + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-bridge-ledger-ok", + target, + runtimeRootDir, + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: `http://127.0.0.1:${address.port}`, + maxBodyBytes: 512, + }); + try { + const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, { + method: "GET", + headers: { + authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, + }, + }); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toEqual({ id: "issue-1" }); + // The reader released every token, so the aggregate gauge and the live-token + // registry both return to zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + await bridge?.stop(); + await new Promise((resolve) => apiServer.close(() => resolve())); + } + }); + + it("fails a response-body read closed when the host aggregate ledger has no room and retains no bytes", async () => { + // The aggregate ledger sits at a tiny ceiling, so a response body larger than + // the ceiling cannot reserve its bytes. The reader fails closed: it cancels + // the stream reader, retains nothing, and reports the fixed marker. The safe + // GET maps the marker to a retryable 502. The ledger returns to zero, because + // the reader released the tokens it held before the rejection. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-execution-target-bridge-ledger-full-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + const runtimeRootDir = path.join(remoteCwd, ".paperclip-runtime", "codex"); + await mkdir(runtimeRootDir, { recursive: true }); + + // The body sits under the per-request size limit but over the aggregate + // ceiling, so the aggregate ledger, not the per-request limit, rejects it. + const responseBody = "x".repeat(256); + const apiServer = createServer((_req, res) => { + res.writeHead(200, { + "content-type": "application/json", + "content-length": String(Buffer.byteLength(responseBody, "utf8")), + }); + res.end(responseBody); + }); + await new Promise((resolve, reject) => { + apiServer.once("error", reject); + apiServer.listen(0, "127.0.0.1", () => resolve()); + }); + const address = apiServer.address(); + if (!address || typeof address === "string") { + throw new Error("Expected the bridge test API server to listen on a TCP port."); + } + + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 8 }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "e2b", + environmentId: "env-1", + leaseId: "lease-1", + remoteCwd, + runner: createLocalSandboxRunner(), + timeoutMs: 30_000, + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-bridge-ledger-full", + target, + runtimeRootDir, + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: `http://127.0.0.1:${address.port}`, + maxBodyBytes: 4096, + }); + try { + const response = await fetch(`${bridge!.env.PAPERCLIP_API_URL}/api/issues/issue-1`, { + method: "GET", + headers: { + authorization: `Bearer ${bridge!.env.PAPERCLIP_API_KEY}`, + }, + }); + + // The safe GET maps the aggregate rejection to a retryable 502 with no + // indeterminate marker. The body carries only the fixed rejection marker. + expect(response.status).toBe(502); + expect(response.headers.get("x-paperclip-bridge-outcome")).toBeNull(); + await expect(response.json()).resolves.toEqual({ + error: DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + }); + // The reader released the tokens it held before the rejection, so the + // aggregate gauge and the live-token registry both return to zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + await bridge?.stop(); + await new Promise((resolve) => apiServer.close(() => resolve())); + } + }); + it("forwards the host indeterminate-outcome header so the sandbox server maps the 504 to a non-retryable 409", async () => { // The host marks a possibly-committed mutation with a 504 and the // `x-paperclip-bridge-outcome: indeterminate` header. The forward must keep @@ -2981,7 +3186,7 @@ describe("sandbox adapter execution targets", () => { if (onOpen) { onOpen({ nonce, port, emitRaw, emitFrame, emitExit }); } else { - emitFrame({ version: 1, type: "ready", nonce }); + emitFrame({ version: 2, type: "ready", nonce }); } }); }, @@ -3091,8 +3296,9 @@ describe("sandbox adapter execution targets", () => { // The gateway forwards one agent request as a request frame. The broker // forwards it on the host path with the real token and the run id, then // writes one response frame back. - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-1", @@ -3100,8 +3306,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -3130,6 +3336,66 @@ describe("sandbox adapter execution targets", () => { expect(control.stopCount).toBeGreaterThanOrEqual(1); }, 20000); + it("falls back to the file bridge when a post-READY pre-bind flood exceeds the aggregate ceiling", async () => { + // The gateway sends a valid READY, then floods the channel before the broker + // binds. The pre-READY buffer cap does not bound the post-READY replay buffer, + // so the replay reservation must. The ceiling admits the small READY frame but + // rejects the flood. The host drops the buffer, stops the channel, and selects + // the file bridge with the aggregate marker. No request forwards, and the + // aggregate ledger returns to zero. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-replay-flood-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + await mkdir(remoteCwd, { recursive: true }); + const api = await startRecordingApiServer(); + // The flood is larger than the ceiling; the READY frame is far smaller. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 4096 }); + const { runner, control } = makeDuplexSelectionRunner((ctx) => { + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); + ctx.emitRaw("x".repeat(64 * 1024)); + }); + const { recorder, counters } = createRecordingDuplexRecorder(); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "daytona", + remoteCwd, + timeoutMs: 30_000, + runner, + effectiveCapabilities: duplexCapabilities(true), + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-duplex-replay-flood", + target, + runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: api.origin, + enableSandboxDuplexBridge: true, + duplexTelemetryRecorder: recorder, + }); + try { + // The file bridge serves, not the duplex transport. + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1"); + // The fallback names the aggregate marker on the file transport. + const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL); + expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded"); + expect(fallback?.dimensions.transport).toBe("file"); + // The gate stopped the flooded channel. + expect(control.stopCount).toBeGreaterThanOrEqual(1); + // No request forwarded, because the broker never bound. + expect(api.requests).toHaveLength(0); + // The aggregate ledger returns to zero with no live token. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + await bridge?.stop(); + await api.close(); + } + }, 20000); + it("streams run logs on the duplex path under the same gate and log line as the file path", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-runlog-")); cleanupDirs.push(rootDir); @@ -3338,15 +3604,15 @@ describe("sandbox adapter execution targets", () => { { name: "a mismatched nonce", onOpen: (ctx: DuplexOpenContext) => - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }), + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }), }, { name: "an incomplete READY frame", - onOpen: (ctx: DuplexOpenContext) => ctx.emitRaw('{"version":1,"type":"ready"}\n'), + onOpen: (ctx: DuplexOpenContext) => ctx.emitRaw('{"version":2,"type":"ready"}\n'), }, { name: "protocol contamination before READY", - onOpen: (ctx: DuplexOpenContext) => ctx.emitFrame({ version: 1, type: "heartbeat" }), + onOpen: (ctx: DuplexOpenContext) => ctx.emitFrame({ version: 2, type: "heartbeat" }), }, { name: "a gateway bind failure with no READY frame", @@ -3400,12 +3666,12 @@ describe("sandbox adapter execution targets", () => { { name: "an attacker-owned numeric local port", buildReady: (nonce: string, attackerPort: number) => - `{"version":1,"type":"ready","nonce":"${nonce}","port":${attackerPort}}\n`, + `{"version":2,"type":"ready","nonce":"${nonce}","port":${attackerPort}}\n`, }, { name: "a channel-supplied host URL", buildReady: (nonce: string, attackerPort: number) => - `{"version":1,"type":"ready","nonce":"${nonce}","address":"http://127.0.0.1:${attackerPort}"}\n`, + `{"version":2,"type":"ready","nonce":"${nonce}","address":"http://127.0.0.1:${attackerPort}"}\n`, }, ])( "rejects a READY frame that carries $name and never sends the bridge token there", @@ -3505,8 +3771,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-forbidden", @@ -3514,8 +3781,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/secret-admin-route", query: "", headers: { authorization: "Bearer bridge-token", "content-type": "application/json" }, - body: JSON.stringify({ escalate: true }), - }), + }, + JSON.stringify({ escalate: true }), ); await waitForCondition( () => control.written.length >= 1, @@ -3583,6 +3850,24 @@ describe("sandbox adapter execution targets", () => { ]); }); + it("pins the exact aggregate byte ledger metric names", () => { + // The aggregate byte ledger metric names are closed. This test locks the + // exact set, so a new gauge or counter name needs an explicit change here. + // Each record carries only closed constant dimensions and no dynamic label. + expect([...DUPLEX_AGGREGATE_BYTE_LEDGER_METRIC_NAMES]).toEqual([ + "sandbox_duplex_aggregate_bytes_in_use", + "sandbox_duplex_aggregate_byte_reservation_rejections_total", + "sandbox_duplex_aggregate_byte_accounting_underflow_total", + ]); + expect(DUPLEX_GAUGE_AGGREGATE_BYTES_IN_USE).toBe("sandbox_duplex_aggregate_bytes_in_use"); + expect(DUPLEX_COUNTER_AGGREGATE_BYTE_RESERVATION_REJECTIONS_TOTAL).toBe( + "sandbox_duplex_aggregate_byte_reservation_rejections_total", + ); + expect(DUPLEX_COUNTER_AGGREGATE_BYTE_ACCOUNTING_UNDERFLOW_TOTAL).toBe( + "sandbox_duplex_aggregate_byte_accounting_underflow_total", + ); + }); + it("records a duplex request span with latency and the fixed dimension keys", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-obs-")); cleanupDirs.push(rootDir); @@ -3613,8 +3898,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-obs", @@ -3622,8 +3908,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -3701,6 +3987,7 @@ describe("sandbox adapter execution targets", () => { "ready_nonce_mismatch", "ready_timeout", "contaminated", + "aggregate_bytes_exceeded", ]; expect(approvedReasons).toContain(fallback?.dimensions.fallback_reason); expect(fallback?.dimensions).toMatchObject({ transport: "file", outcome: "error" }); @@ -3822,8 +4109,9 @@ describe("sandbox adapter execution targets", () => { try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); if (dispatchFirst) { - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-loss", @@ -3831,8 +4119,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -3888,8 +4176,9 @@ describe("sandbox adapter execution targets", () => { try { // The throwing recorder never blocked the duplex selection. expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-guard", @@ -3897,8 +4186,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -3954,7 +4243,7 @@ describe("sandbox adapter execution targets", () => { onData(listener: (chunk: string) => void): void { dataListener = listener; control.emitData = (chunk) => dataListener?.(chunk); - setImmediate(() => dataListener?.(`${JSON.stringify({ version: 1, type: "ready", nonce })}\n`)); + setImmediate(() => dataListener?.(`${JSON.stringify({ version: 2, type: "ready", nonce })}\n`)); }, onExit(_listener: (exit: { exitCode: number | null }) => void): void {}, stop(): void {}, @@ -3999,8 +4288,9 @@ describe("sandbox adapter execution targets", () => { // A request that carries the sentinel route, query, body, and bridge token. // The response write then fails with the sentinel provider error. - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-redact", @@ -4008,8 +4298,8 @@ describe("sandbox adapter execution targets", () => { path: `/api/${ROUTE_SENTINEL}`, query: `secret=${QUERY_SENTINEL}`, headers: { authorization: `Bearer ${BRIDGE_TOKEN_SENTINEL}` }, - body: BODY_SENTINEL, - }), + }, + BODY_SENTINEL, ); // Give the forward and the failing response write time to run and record a loss. await waitForCondition( @@ -4065,8 +4355,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-prov", @@ -4074,8 +4365,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => spans.some((s) => s.name === DUPLEX_SPAN_REQUEST), @@ -4213,6 +4504,128 @@ describe("sandbox adapter execution targets", () => { } }, 20000); + it("fails the readiness handshake closed when the host aggregate ledger has no room for the pre-READY buffer", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-ready-ledger-full-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + await mkdir(remoteCwd, { recursive: true }); + const api = await startRecordingApiServer(); + // The host stamps one process-owned aggregate byte ledger on the sandbox + // target at a tiny ceiling. The fake gateway sends a pre-READY blob larger + // than the ceiling, so the gate cannot reserve the blob bytes. The gate fails + // closed: it retains nothing, records the aggregate fallback reason, and falls + // back to the file bridge. The blob is smaller than the readiness buffer cap, + // so the aggregate ledger, not the buffer cap, drives the failure. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 256 }); + const preReadyBlob = "x".repeat(4_096); + const { runner, control } = makeDuplexSelectionRunner((ctx) => { + ctx.emitRaw(preReadyBlob); + }); + const { recorder, counters } = createRecordingDuplexRecorder(); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "daytona", + remoteCwd, + timeoutMs: 30_000, + runner, + effectiveCapabilities: duplexCapabilities(true), + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-ready-ledger-full", + target, + runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: api.origin, + enableSandboxDuplexBridge: true, + // A long readiness timeout, so the aggregate ledger, not the timeout, drives + // the failure. + duplexReadinessTimeoutMs: 5_000, + duplexTelemetryRecorder: recorder, + }); + try { + expect(bridge).not.toBeNull(); + expect(control.openCount).toBe(1); + // The aggregate rejection drove the failure, so the file bridge serves. + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("queue_v1"); + const fallback = counters.find((c) => c.metric === DUPLEX_COUNTER_FALLBACK_TOTAL); + expect(fallback?.dimensions.fallback_reason).toBe("aggregate_bytes_exceeded"); + // The gate retained nothing after the rejection, so the aggregate gauge and + // the live-token registry both return to zero. + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(control.closeCount + control.stopCount).toBeGreaterThanOrEqual(1); + } finally { + await bridge?.stop(); + await api.close(); + } + }, 20000); + + it("charges the pre-READY buffer against the injected host ledger and releases it when readiness passes", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-ready-ledger-ok-")); + cleanupDirs.push(rootDir); + const remoteCwd = path.join(rootDir, "workspace"); + await mkdir(remoteCwd, { recursive: true }); + const api = await startRecordingApiServer(); + // The host stamps one process-owned aggregate byte ledger on the sandbox + // target at a generous ceiling. The fake gateway sends one pre-READY noise + // line, then the valid READY frame. The gate charges the noise bytes against + // the injected ledger, passes readiness, and releases the pre-READY tokens. + // The broker's frame decoder re-charges any post-READY bytes, so the ledger + // returns to zero after the handshake. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 * 1024 }); + const reserveSpy = vi.spyOn(ledger, "reserve"); + const { runner, control } = makeDuplexSelectionRunner((ctx) => { + ctx.emitRaw("pty-echo-noise"); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); + }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "daytona", + remoteCwd, + timeoutMs: 30_000, + runner, + effectiveCapabilities: duplexCapabilities(true), + duplexAggregateByteLedger: ledger, + }; + + const bridge = await startAdapterExecutionTargetPaperclipBridge({ + runId: "run-ready-ledger-ok", + target, + runtimeRootDir: path.join(remoteCwd, ".paperclip-runtime", "codex"), + adapterKey: "codex", + hostApiToken: "real-run-jwt", + hostApiUrl: api.origin, + enableSandboxDuplexBridge: true, + duplexReadinessTimeoutMs: 5_000, + }); + try { + expect(bridge).not.toBeNull(); + // Readiness passed, so the duplex transport serves. + expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); + // The gate charged the pre-READY noise against the exact injected ledger, so + // the identity holds at this seam. + expect(reserveSpy).toHaveBeenCalledWith("readiness_buffer", expect.any(Number)); + // The gate released every readiness-buffer token on settle, so the aggregate + // gauge and the live-token registry both return to zero. + await waitForCondition( + () => ledger.bytesInUse === 0 && ledger.liveTokenCount === 0, + "the readiness gate to release every pre-READY token", + 4000, + ); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + } finally { + reserveSpy.mockRestore(); + await bridge?.stop(); + await api.close(); + } + }, 20000); + it("bounds the pre-READY newline-scan work by the bytes received", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-duplex-scan-")); cleanupDirs.push(rootDir); @@ -4284,9 +4697,9 @@ describe("sandbox adapter execution targets", () => { const noisePrefix = "\n".repeat(blankLineCount) + "a non-frame echo line\n"; const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw(noisePrefix); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); - const readyLine = '{"version":1,"type":"ready","nonce":""}\n'; + const readyLine = '{"version":2,"type":"ready","nonce":""}\n'; const totalBytes = noisePrefix.length + readyLine.length; const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4341,8 +4754,8 @@ describe("sandbox adapter execution targets", () => { // must skip the echo line and a partial-JSON line, then accept the READY frame. const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("sh -c exec env PAPERCLIP_BRIDGE_NONCE=... node gateway.mjs\n"); - ctx.emitRaw('{"version":1,"type":"ready"}\n'); - ctx.emitFrame({ version: 1, type: "ready", nonce: ctx.nonce }); + ctx.emitRaw('{"version":2,"type":"ready"}\n'); + ctx.emitFrame({ version: 2, type: "ready", nonce: ctx.nonce }); }); const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4392,7 +4805,7 @@ describe("sandbox adapter execution targets", () => { // the `ready_nonce_mismatch` reason. const { runner, control } = makeDuplexSelectionRunner((ctx) => { ctx.emitRaw("a non-frame echo line\n"); - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }); + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }); }); const { recorder, counters } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4448,7 +4861,7 @@ describe("sandbox adapter execution targets", () => { // per-skip cap check stops the bypass. const readinessBufferCapBytes = DEFAULT_MAX_DUPLEX_FRAME_BYTES + 4_096; const { runner, control } = makeDuplexSelectionRunner((ctx) => { - const readyLine = `${JSON.stringify({ version: 1, type: "ready", nonce: ctx.nonce })}\n`; + const readyLine = `${JSON.stringify({ version: 2, type: "ready", nonce: ctx.nonce })}\n`; ctx.emitRaw("\n".repeat(readinessBufferCapBytes + 1) + readyLine); }); const { recorder, counters } = createRecordingDuplexRecorder(); @@ -4500,7 +4913,7 @@ describe("sandbox adapter execution targets", () => { // carries the closed `fallback_reason` dimension, so a reader can group the // failed opens by reason. const { runner } = makeDuplexSelectionRunner((ctx) => - ctx.emitFrame({ version: 1, type: "ready", nonce: "00000000000000000000000000000000" }), + ctx.emitFrame({ version: 2, type: "ready", nonce: "00000000000000000000000000000000" }), ); const { recorder, spans } = createRecordingDuplexRecorder(); const target: AdapterSandboxExecutionTarget = { @@ -4570,8 +4983,9 @@ describe("sandbox adapter execution targets", () => { }); try { expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-hdr", @@ -4586,8 +5000,8 @@ describe("sandbox adapter execution targets", () => { // A sandbox-supplied auth header the host must replace with the real token. authorization: "Bearer bridge-token", }, - body: "", - }), + }, + "", ); await waitForCondition( () => api.requests.length >= 1, @@ -4640,8 +5054,9 @@ describe("sandbox adapter execution targets", () => { try { // The broker started with derived nested budgets, so the duplex transport serves. expect(bridge?.env.PAPERCLIP_API_BRIDGE_MODE).toBe("duplex_v1"); - control.emitData( - encodeDuplexFrame({ + emitDuplexRequest( + control, + { version: DUPLEX_FRAME_VERSION, type: "request", id: "req-budget", @@ -4649,8 +5064,8 @@ describe("sandbox adapter execution targets", () => { path: "/api/agents/me", query: "", headers: { authorization: "Bearer bridge-token" }, - body: "", - }), + }, + "", ); await waitForCondition( () => control.written.length >= 1, @@ -4760,7 +5175,7 @@ describe("sandbox adapter execution targets", () => { "daytona@212487a7f3c9:~$ exec 2>'/tmp/paperclip-duplex-x.log'; stty raw -echo; " + "exec 'bash' '-c' 'exec env PAPERCLIP_BRIDGE_NONCE=" + ctx.nonce + " node gateway.mjs'\r\n", ); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -4777,7 +5192,7 @@ describe("sandbox adapter execution targets", () => { "stty raw -echo; exec 'bash' '-c' 'exec env node gateway.mjs'\r\n", ); // No newline between the escape sequence and the frame: same line. - ctx.emitRaw('\x1b[?2004l\r{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('\x1b[?2004l\r{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -4786,7 +5201,7 @@ describe("sandbox adapter execution targets", () => { it("PTY replay: accepts READY split across chunk boundaries", async () => { const { mode } = await runReadinessReplay((ctx) => { ctx.emitRaw("prompt$ wrapper-line\r\n"); - const frame = '{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'; + const frame = '{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'; ctx.emitRaw(frame.slice(0, 12)); ctx.emitRaw(frame.slice(12)); }); @@ -4799,7 +5214,7 @@ describe("sandbox adapter execution targets", () => { const noise = Buffer.from("prompt ✓ done\r\n", "utf8"); ctx.emitRaw(noise.slice(0, 8).toString("utf8")); ctx.emitRaw(noise.slice(8).toString("utf8")); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("duplex_v1"); }, 20000); @@ -4810,7 +5225,7 @@ describe("sandbox adapter execution targets", () => { const { mode } = await runReadinessReplay((ctx) => { const line = "x".repeat(64) + "\n"; for (let i = 0; i < 80_000; i += 1) ctx.emitRaw(line); - ctx.emitRaw('{"version":1,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); + ctx.emitRaw('{"version":2,"type":"ready","nonce":"' + ctx.nonce + '"}\n'); }); expect(mode).toBe("queue_v1"); }, 30000); @@ -4842,14 +5257,24 @@ interface EmbeddedDecodeResult { // The names the embedded codec source declares. A test wraps the source and // reads these names back. +type EmbeddedEncodeResult = + | { ok: true; line: string } + | { ok: false; error: { code: string; message: string } }; + interface EmbeddedCodec { encodeDuplexFrame: (frame: unknown) => string; + encodeDuplexFrameChecked: (frame: unknown, maxFrameBytes?: number) => EmbeddedEncodeResult; decodeDuplexLine: (line: string | Buffer) => EmbeddedDecodeResult; - DuplexFrameDecoder: new (options?: { maxFrameBytes?: number }) => { + DuplexFrameDecoder: new (options?: { maxFrameBytes?: number; maxAggregateBytes?: number }) => { push: (chunk: Buffer) => EmbeddedDecodeResult[]; + scope: string; + bytesInUse: number; + aggregateRejections: number; }; DUPLEX_FRAME_VERSION: number; DEFAULT_MAX_DUPLEX_FRAME_BYTES: number; + DUPLEX_DECODER_SCOPE: string; + DEFAULT_MAX_DUPLEX_DECODER_BYTES: number; } type ExpectedVectorResult = { frame: unknown } | { error: string }; @@ -4864,10 +5289,18 @@ interface DuplexFrameVector { expected: ExpectedVectorResult[]; } +interface DuplexEncodeVector { + name: string; + maxFrameBytes: number; + frame: unknown; + expected: { ok: true } | { ok: false; error: string }; +} + interface DuplexFrameFixture { frameVersion: number; defaultMaxFrameBytes: number; vectors: DuplexFrameVector[]; + encodeVectors: DuplexEncodeVector[]; } describe("sandbox duplex gateway", () => { @@ -4902,7 +5335,7 @@ describe("sandbox duplex gateway", () => { // process the launch replaced the shell with. await writeFile( stub, - '#!/bin/sh\nprintf \'{"version":1,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', + '#!/bin/sh\nprintf \'{"version":2,"type":"ready","nonce":"%s"}\\n\' "$PAPERCLIP_BRIDGE_NONCE"\n', "utf8", ); const argv = buildDuplexGatewayLaunchArgv({ @@ -5086,7 +5519,7 @@ describe("sandbox duplex gateway", () => { it("embedded gateway codec passes every vector in the shared fixture", async () => { const codecFactory = new Function( - `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES };`, + `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, encodeDuplexFrameChecked, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`, ) as unknown as () => EmbeddedCodec; const codec = codecFactory(); @@ -5152,6 +5585,93 @@ describe("sandbox duplex gateway", () => { expect(decoded.ok).toBe(true); expect(decoded.frame).toEqual(want.frame); } + + // The embedded copy enforces the same encode bound as the host copy. It runs + // every shared encode vector and matches the expected result, so both copies + // reject the same oversized frame with the same `frame_too_large` code. + expect(fixture.encodeVectors.length).toBeGreaterThanOrEqual(3); + const encodeFailures: string[] = []; + for (const vector of fixture.encodeVectors) { + const result = codec.encodeDuplexFrameChecked(vector.frame, vector.maxFrameBytes); + if (result.ok !== vector.expected.ok) { + encodeFailures.push(`${vector.name}: ok=${result.ok}, want ${vector.expected.ok}`); + continue; + } + if (result.ok) { + if (!result.line.endsWith("\n") || result.line.slice(0, -1).includes("\n")) { + encodeFailures.push(`${vector.name}: line is not exactly one frame line`); + continue; + } + const decoded = codec.decodeDuplexLine(result.line.slice(0, -1)); + if (!decoded.ok) { + encodeFailures.push(`${vector.name}: an ok encode did not decode back`); + continue; + } + try { + expect(decoded.frame).toEqual(vector.frame); + } catch { + encodeFailures.push(`${vector.name}: decoded frame does not match`); + } + } else if (!vector.expected.ok && result.error.code !== vector.expected.error) { + encodeFailures.push(`${vector.name}: error ${result.error.code} != ${vector.expected.error}`); + } + } + expect(encodeFailures).toEqual([]); + }); + + it("bounds the sandbox decoder with a separate sandbox_process scope, distinct from the host ledger scope", () => { + // The generated gateway runs in a separate operating-system process, so its + // decoder cannot share the host aggregate byte ledger. It enforces a separate + // local cap under the `sandbox_process` scope. This test wraps the embedded + // source and the host decoder, then proves the two scopes never overlap and the + // sandbox cap fails closed. + const codecFactory = new Function( + `${getSandboxDuplexGatewayCodecSource()}\nreturn { encodeDuplexFrame, decodeDuplexLine, DuplexFrameDecoder, DUPLEX_FRAME_VERSION, DEFAULT_MAX_DUPLEX_FRAME_BYTES, DUPLEX_DECODER_SCOPE, DEFAULT_MAX_DUPLEX_DECODER_BYTES };`, + ) as unknown as () => EmbeddedCodec; + const codec = codecFactory(); + + // The sandbox scope is the fixed `sandbox_process` label. + expect(codec.DUPLEX_DECODER_SCOPE).toBe("sandbox_process"); + expect(codec.DEFAULT_MAX_DUPLEX_DECODER_BYTES).toBeGreaterThan(codec.DEFAULT_MAX_DUPLEX_FRAME_BYTES); + // The two scopes are distinct: the host owner set never carries the sandbox + // scope, so the sandbox counter can never map to a host aggregate token. + expect((DUPLEX_AGGREGATE_TOKEN_OWNERS as readonly string[]).includes("sandbox_process")).toBe(false); + + // The sandbox decoder tracks its own `sandbox_process` counter. A frame under + // the cap charges the local counter, and the counter returns to zero once the + // frame drains. + const sandboxDecoder = new codec.DuplexFrameDecoder({ maxAggregateBytes: 64 }); + expect(sandboxDecoder.scope).toBe("sandbox_process"); + const partial = sandboxDecoder.push(Buffer.from('{"version":1,', "utf8")); + expect(partial).toEqual([]); + expect(sandboxDecoder.bytesInUse).toBe(Buffer.byteLength('{"version":1,', "utf8")); + sandboxDecoder.push(Buffer.from('"type":"heartbeat"}\n', "utf8")); + expect(sandboxDecoder.bytesInUse).toBe(0); + + // A chunk over the local cap fails closed. The decoder retains nothing, reports + // the aggregate rejection, and increments only its local `sandbox_process` + // counter. + const cappedDecoder = new codec.DuplexFrameDecoder({ maxAggregateBytes: 8 }); + const overCap = cappedDecoder.push(Buffer.from("x".repeat(64), "utf8")); + expect(overCap.length).toBe(1); + const rejection = overCap[0]; + expect(rejection.ok).toBe(false); + expect(rejection.error?.code).toBe("aggregate_bytes_exceeded"); + expect(cappedDecoder.bytesInUse).toBe(0); + expect(cappedDecoder.aggregateRejections).toBe(1); + + // The host decoder charges the host aggregate byte ledger under the + // `decoder_buffer` owner. It never uses the sandbox scope. This proves the two + // implementations use distinct scopes: the host uses the injected ledger; the + // sandbox uses its local counter. + const ledger = new DuplexAggregateByteLedger({ ceilingBytes: 1024 }); + const hostDecoder = new DuplexFrameDecoder({ aggregateByteLedger: ledger }); + hostDecoder.push(Buffer.from('{"version":1,', "utf8")); + expect(ledger.bytesInUse).toBe(Buffer.byteLength('{"version":1,', "utf8")); + expect(ledger.liveTokenCount).toBeGreaterThan(0); + hostDecoder.dispose(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); }); it("returns the same HTTP response as the file gateway for a forwarded request", async () => { @@ -5176,15 +5696,16 @@ describe("sandbox duplex gateway", () => { "if-none-match": '"cache-key"', }); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, - body: JSON.stringify({ ok: true }), - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json", etag: '"rev-1"', "content-length": "999" }, + outcome: "completed", + }, + JSON.stringify({ ok: true }), + ); const response = await responsePromise; expect(response.status).toBe(200); expect(response.headers.get("content-type")).toContain("application/json"); @@ -5201,21 +5722,74 @@ describe("sandbox duplex gateway", () => { const patchFrame = await gateway.waitForFrame( (frame) => frame.type === "request" && frame.id !== requestFrame.id, ); - gateway.sendFrame({ - version: 1, - type: "response", - id: patchFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: JSON.stringify({ error: "outcome_indeterminate" }), - outcome: "indeterminate", - }); + sendDuplexResponse( + gateway.sendFrame, + { + id: patchFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "indeterminate", + }, + JSON.stringify({ error: "outcome_indeterminate" }), + ); const indeterminate = await indeterminatePromise; expect(indeterminate.status).toBe(409); await gateway.stop(); }, 20000); + it("fails a request over the body size limit locally and keeps the channel open", async () => { + const token = "duplex-token-oversize-request"; + // The chunked body protocol splits a large body into fixed-size body_chunk + // frames that each stay under the frame bound, so a body never exceeds the + // frame bound on its own. The operative local guard for a request too large to + // deliver is now the body size limit. Set it to 1,000,000 bytes here. + const gateway = await startDuplexGateway({ + PAPERCLIP_BRIDGE_TOKEN: token, + PAPERCLIP_BRIDGE_MAX_BODY_BYTES: "1000000", + }); + + // A body over the body size limit is rejected locally. The gateway must fail + // this one local request cleanly and forward no frame. + const oversizeBody = JSON.stringify({ body: "x".repeat(1_100_000) }); + const tooLarge = await fetch(`${gateway.baseUrl}/api/issues/issue-1/comments`, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: oversizeBody, + }); + expect(tooLarge.status).toBe(502); + await expect(tooLarge.json()).resolves.toEqual({ + error: "Bridge request body exceeded the configured size limit.", + }); + + // The oversized request forwarded no frame. It never left the gateway. + expect(gateway.frames.filter((frame) => frame.type === "request")).toHaveLength(0); + // The gateway did not lose the channel: no close frame and the process runs. + expect(gateway.frames.some((frame) => frame.type === "close")).toBe(false); + + // The channel stays open, so a normal request after the oversized one still + // forwards and completes. + const okPromise = fetch(`${gateway.baseUrl}/api/agents/me`, { + headers: { authorization: `Bearer ${token}` }, + }); + const requestFrame = await gateway.waitForFrame((frame) => frame.type === "request"); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "completed", + }, + JSON.stringify({ ok: true }), + ); + const okResponse = await okPromise; + expect(okResponse.status).toBe(200); + await expect(okResponse.json()).resolves.toEqual({ ok: true }); + + await gateway.stop(); + }, 20000); + it("enforces the bearer check, JSON-only rule, body limit, and depth limit", async () => { const token = "duplex-token-contract"; const gateway = await startDuplexGateway({ @@ -5345,24 +5919,21 @@ describe("sandbox duplex gateway", () => { // Feed a malformed inbound line and an unknown response id. Both force a // diagnostic path; none of it may reach stdout. gateway.sendRaw("this is not a frame\n"); - gateway.sendFrame({ - version: 1, - type: "response", - id: "unknown-id", - status: 200, - headers: {}, - body: "", - outcome: "completed", - }); - gateway.sendFrame({ - version: 1, - type: "response", - id: requestFrame.id, - status: 200, - headers: { "content-type": "application/json" }, - body: "{}", - outcome: "completed", - }); + sendDuplexResponse( + gateway.sendFrame, + { id: "unknown-id", status: 200, headers: {}, outcome: "completed" }, + "", + ); + sendDuplexResponse( + gateway.sendFrame, + { + id: requestFrame.id, + status: 200, + headers: { "content-type": "application/json" }, + outcome: "completed", + }, + "{}", + ); const response = await responsePromise; expect(response.status).toBe(200); @@ -5418,7 +5989,7 @@ function createFakeDuplexChannel(): { channel: CommandManagedDuplexChannel; emitData: (chunk: string) => void; emitExit: (exit: { exitCode: number | null }) => void; - written: DuplexResponseFrame[]; + written: Array; writtenTypes: string[]; stopped: () => number; setWriteError: (error: Error | null) => void; @@ -5429,16 +6000,43 @@ function createFakeDuplexChannel(): { let writeError: Error | null = null; let closeBehavior: "resolve" | "hang" | "reject" = "resolve"; let stopCount = 0; - const written: DuplexResponseFrame[] = []; + // The broker writes a response as one envelope frame that carries + // `bodyByteCount`, then the `body_chunk` frames that carry the body. Reassemble + // the body so a test reads the response the same way it did with the one-frame + // model. + const written: Array = []; const writtenTypes: string[] = []; + const responseAssembly = new Map< + string, + { frame: DuplexResponseFrame; received: number; chunks: Buffer[] } + >(); const channel: CommandManagedDuplexChannel = { write(data: string): void { if (writeError) throw writeError; const decoded = decodeDuplexLine(data.replace(/\n$/, "")); - if (decoded.ok) { - writtenTypes.push(decoded.frame.type); - if (decoded.frame.type === "response") written.push(decoded.frame); + if (!decoded.ok) return; + const frame = decoded.frame; + writtenTypes.push(frame.type); + if (frame.type === "response") { + if (frame.bodyByteCount === 0) { + written.push({ ...frame, body: "" }); + } else { + responseAssembly.set(frame.id, { frame, received: 0, chunks: [] }); + } + } else if (frame.type === "body_chunk") { + const assembly = responseAssembly.get(frame.id); + if (!assembly) return; + const decodedChunk = Buffer.from(frame.data, "base64"); + assembly.received += decodedChunk.length; + assembly.chunks.push(decodedChunk); + if (assembly.received >= assembly.frame.bodyByteCount) { + responseAssembly.delete(frame.id); + written.push({ + ...assembly.frame, + body: Buffer.concat(assembly.chunks).toString("utf8"), + }); + } } }, onData(listener: (chunk: string) => void): void { @@ -5473,8 +6071,16 @@ function createFakeDuplexChannel(): { }; } -/** Build one request frame line the fake channel can emit. */ -function requestFrameLine(overrides: Partial & { id: string }): string { +/** + * Build one request as its envelope line (carrying `bodyByteCount`) plus its + * `body_chunk` lines, so the fake channel emits a whole request in one write. The + * broker reassembles the body from the chunk frames before it forwards. + */ +function requestFrameLine( + overrides: Partial> & { id: string }, + bodyText: string = JSON.stringify({ body: "hello" }), +): string { + const body = Buffer.from(bodyText, "utf8"); const frame: DuplexRequestFrame = { version: DUPLEX_FRAME_VERSION, type: "request", @@ -5483,9 +6089,13 @@ function requestFrameLine(overrides: Partial & { id: string path: overrides.path ?? "/api/issues/PAP-1/comments", query: overrides.query ?? "", headers: overrides.headers ?? { authorization: "Bearer bridge-token" }, - body: overrides.body ?? JSON.stringify({ body: "hello" }), + bodyByteCount: body.length, }; - return encodeDuplexFrame(frame); + let line = encodeDuplexFrame(frame); + for (const chunk of splitBodyIntoChunkFrames(frame.id, body, DUPLEX_FRAME_VERSION)) { + line += encodeDuplexFrame(chunk); + } + return line; } /** Wait for the pending microtasks and macrotasks to settle. */ @@ -5501,7 +6111,7 @@ describe("createDuplexBridgeBroker", () => { // broker passes the decoded request straight through, so the sandbox request // still carries only the bridge token here, and the handler applies the real // token and the signed run identifier on the existing forward path. - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request): Promise => { received.push(request); @@ -5536,7 +6146,7 @@ describe("createDuplexBridgeBroker", () => { it("moves through opening, open, closing, closed in order", async () => { const fake = createFakeDuplexChannel(); const states: DuplexBrokerState[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), onStateChange: (state) => states.push(state), @@ -5575,7 +6185,7 @@ describe("createDuplexBridgeBroker", () => { ])("enters lost on $name", async ({ reason, trigger }) => { const fake = createFakeDuplexChannel(); const losses: DuplexBrokerLossRecord[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), onLoss: (record) => losses.push(record), @@ -5593,7 +6203,7 @@ describe("createDuplexBridgeBroker", () => { it("enters lost on a close timeout", async () => { const fake = createFakeDuplexChannel(); fake.setCloseBehavior("hang"); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), closeTimeoutMs: 20, @@ -5609,7 +6219,7 @@ describe("createDuplexBridgeBroker", () => { it("stops the heartbeat, marks the run bridge ended, and dispatches nothing after loss", async () => { const fake = createFakeDuplexChannel(); const forwarded: string[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request) => { forwarded.push(request.id); @@ -5633,7 +6243,7 @@ describe("createDuplexBridgeBroker", () => { it("forwards one request id one time, so a repeated frame never reaches the API twice", async () => { const fake = createFakeDuplexChannel(); const forwarded: string[] = []; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async (request) => { forwarded.push(request.id); @@ -5657,7 +6267,7 @@ describe("createDuplexBridgeBroker", () => { const fake = createFakeDuplexChannel(); const records: DuplexBrokerRequestRecord[] = []; let clock = 1000; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), now: () => clock, @@ -5680,7 +6290,7 @@ describe("createDuplexBridgeBroker", () => { it("latches a failure when a loss orders before an orderly completion and names the typed reason", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -5713,7 +6323,7 @@ describe("createDuplexBridgeBroker", () => { }, }; const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), telemetry: createDuplexTelemetry({ recorder, providerKey: "daytona" }), @@ -5749,7 +6359,7 @@ describe("createDuplexBridgeBroker", () => { const logLines: string[] = []; const fake = createFakeDuplexChannel(); const sentinel = "SENTINEL-PROVIDER-TEXT-1a2b3c"; - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), telemetry: createDuplexTelemetry({ recorder, providerKey: "daytona" }), @@ -5773,7 +6383,7 @@ describe("createDuplexBridgeBroker", () => { expect(logLines.join("\n")).not.toContain(sentinel); }); - it("rejects a configuration where an inner budget is not smaller than its outer budget", () => { + it("rejects a configuration where an inner budget is not smaller than its outer budget", async () => { expect(() => assertNestedDuplexBrokerBudgets({ forwardTimeoutMs: 32_000, @@ -5788,13 +6398,15 @@ describe("createDuplexBridgeBroker", () => { gatewayWaitMs: 35_000, }), ).toThrow(/response budget/); - expect(() => + // The async broker construction validates budgets before its first await, so + // an invalid budget set surfaces as a rejected promise. + await expect( createDuplexBridgeBroker({ channel: createFakeDuplexChannel().channel, forwardRequest: async () => ({ status: 200 }), budgets: { forwardTimeoutMs: 40_000 }, }), - ).toThrow(/forward budget/); + ).rejects.toThrow(/forward budget/); // The default budget set holds the nested order. expect(() => assertNestedDuplexBrokerBudgets({ @@ -5870,7 +6482,7 @@ describe("sandbox target spec parse: enableSandboxDuplexBridge", () => { describe("settleRunDisposition atomic read and mark", () => { it("marks the orderly completion and reports a success for a healthy channel", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -5886,7 +6498,7 @@ describe("settleRunDisposition atomic read and mark", () => { it("reports the failure and does not mark for a latched loss", async () => { const fake = createFakeDuplexChannel(); - const broker = createDuplexBridgeBroker({ + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -5929,8 +6541,8 @@ describe("CLI-lane run-disposition seam", () => { } as AdapterSandboxExecutionTarget; } - function startBroker(fake: ReturnType) { - const broker = createDuplexBridgeBroker({ + async function startBroker(fake: ReturnType) { + const broker = await createDuplexBridgeBroker({ channel: fake.channel, forwardRequest: async () => ({ status: 200 }), }); @@ -5940,7 +6552,7 @@ describe("CLI-lane run-disposition seam", () => { it("fails a clean CLI completion closed when the duplex channel was lost mid-turn", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The control channel dies mid-turn, before the CLI process exits. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -5964,7 +6576,7 @@ describe("CLI-lane run-disposition seam", () => { it("keeps a clean CLI completion a success when the channel stays healthy, and a teardown loss stays benign", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); const runner = mockRunner({ ...CLEAN_RESULT }); const result = await runAdapterExecutionTargetProcess("run-cli-ok", sandboxTarget(runner), "agent-cli", [], { @@ -5988,7 +6600,7 @@ describe("CLI-lane run-disposition seam", () => { it("keeps a clean CLI completion a success when the gateway exits during the run-log tail finish", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // A run-log tail whose finish emits a gateway exit. This reproduces the // race where the duplex gateway dies after the clean process completion but @@ -6028,7 +6640,7 @@ describe("CLI-lane run-disposition seam", () => { it("cannot clear the loss latch with a later completion", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The loss latches before the CLI process exits. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6051,7 +6663,7 @@ describe("CLI-lane run-disposition seam", () => { it("leaves an already-failed CLI result unchanged and never settles the disposition", async () => { const fake = createFakeDuplexChannel(); - const broker = startBroker(fake); + const broker = await startBroker(fake); // The control channel is lost, but the process itself also exited non-zero. fake.emitExit({ exitCode: 1 }); await flushMacrotasks(); @@ -6078,3 +6690,213 @@ describe("CLI-lane run-disposition seam", () => { expect(settleCalls).toBe(0); }); }); + +describe("duplex readiness gate replay-buffer reservation", () => { + const READY_NONCE = "0123456789abcdef0123456789abcdef"; + + // A fake duplex channel the test drives directly. `control.emitData` re-enters + // the data listener the gate bound at construction. `control.emitExit` re-enters + // the exit listener. The fake records the stop and the close calls. + function makeFakeReadinessChannel(): { + channel: CommandManagedDuplexChannel; + control: { + stopCount: number; + closeCount: number; + written: string[]; + emitData: (chunk: string) => void; + emitExit: (exit: { exitCode: number | null }) => void; + }; + } { + let dataListener: ((chunk: string) => void) | null = null; + let exitListener: ((exit: { exitCode: number | null }) => void) | null = null; + const control = { + stopCount: 0, + closeCount: 0, + written: [] as string[], + emitData: (chunk: string): void => dataListener?.(chunk), + emitExit: (exit: { exitCode: number | null }): void => exitListener?.(exit), + }; + const channel: CommandManagedDuplexChannel = { + write(data: string): void { + control.written.push(data); + }, + onData(listener: (chunk: string) => void): void { + dataListener = listener; + }, + onExit(listener: (exit: { exitCode: number | null }) => void): void { + exitListener = listener; + }, + stop(): void { + control.stopCount += 1; + }, + close(): Promise { + control.closeCount += 1; + return Promise.resolve(); + }, + }; + return { channel, control }; + } + + // A ledger that counts the reservation-rejection and the accounting-underflow + // signals, so a test proves the one-owner-one-release invariant holds. + function makeCountingLedger(ceilingBytes: number): { + ledger: DuplexAggregateByteLedger; + counts: { rejections: number; underflows: number }; + } { + const counts = { rejections: 0, underflows: 0 }; + const ledger = new DuplexAggregateByteLedger({ + ceilingBytes, + telemetry: { + setBytesInUse(): void {}, + recordReservationRejection(): void { + counts.rejections += 1; + }, + recordAccountingUnderflow(): void { + counts.underflows += 1; + }, + }, + }); + return { ledger, counts }; + } + + function readyLine(): string { + return `${JSON.stringify({ version: 2, type: "ready", nonce: READY_NONCE })}\n`; + } + + it("charges the post-READY suffix and releases it after the broker handoff", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + const suffix = "hello-post-ready-suffix"; + // The READY line and the suffix arrive in one chunk. The gate drops the whole + // pre-READY buffer charge, then charges only the retained suffix. + control.emitData(`${readyLine()}${suffix}`); + const readiness = await gate.ready; + expect(readiness.ok).toBe(true); + expect(gate.replayOverflowed()).toBe(false); + // The gate holds the suffix under one readiness_replay token before the bind. + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + // The broker binds and replays the suffix; the gate releases the token after + // the synchronous handoff. + const replayed: string[] = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + expect(replayed).toEqual([suffix]); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(counts.underflows).toBe(0); + }); + + it("releases the suffix token on disposePendingReplay without a broker handoff", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + const suffix = "abandoned-suffix"; + control.emitData(`${readyLine()}${suffix}`); + expect((await gate.ready).ok).toBe(true); + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + // A broker-construction failure abandons the buffer, so the caller disposes it. + gate.disposePendingReplay(); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + // A second dispose is a no-op and never underflows. + gate.disposePendingReplay(); + expect(ledger.bytesInUse).toBe(0); + expect(counts.underflows).toBe(0); + }); + + it("releases the suffix token after a pre-bind exit then a broker handoff", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + const suffix = "pre-bind-exit-suffix"; + control.emitData(`${readyLine()}${suffix}`); + expect((await gate.ready).ok).toBe(true); + // The channel exits after READY but before the broker binds. The gate holds the + // exit and keeps the pending suffix charged. + control.emitExit({ exitCode: 0 }); + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + // The broker binds, replays the suffix and the exit, then the gate releases the + // reservation. + const replayed: string[] = []; + const exits: Array<{ exitCode: number | null }> = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + gate.brokerChannel.onExit((exit) => exits.push(exit)); + expect(replayed).toEqual([suffix]); + expect(exits).toEqual([{ exitCode: 0 }]); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(counts.underflows).toBe(0); + }); + + it("fails closed when a post-READY pre-bind chunk floods past the ceiling", async () => { + const { channel, control } = makeFakeReadinessChannel(); + // The ceiling admits the small READY line but not the flood chunk. + const { ledger, counts } = makeCountingLedger(256); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + // The READY line arrives alone, so the pending suffix starts empty. + control.emitData(readyLine()); + expect((await gate.ready).ok).toBe(true); + expect(ledger.bytesInUse).toBe(0); + // A post-READY chunk larger than the ceiling floods the replay buffer before + // the broker binds. The gate refuses the reservation and fails closed. + control.emitData("x".repeat(512)); + expect(gate.replayOverflowed()).toBe(true); + expect(control.stopCount).toBe(1); + expect(ledger.bytesInUse).toBe(0); + expect(ledger.liveTokenCount).toBe(0); + expect(counts.rejections).toBe(1); + expect(counts.underflows).toBe(0); + // Binding the broker replays nothing, because the gate dropped the buffer. + const replayed: string[] = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + expect(replayed).toEqual([]); + }); + + it("drops the retained pre-READY buffer on READY acceptance", async () => { + const { channel, control } = makeFakeReadinessChannel(); + const { ledger, counts } = makeCountingLedger(1024 * 1024); + const gate = __duplexReadinessTesting.createReadinessGate(channel, { + nonce: READY_NONCE, + timeoutMs: 5_000, + ledger, + }); + // A large pre-READY noise line and a large suffix arrive with the READY line + // in one chunk. A sandbox controls every byte here. + const noise = `${"n".repeat(4096)}\n`; + const suffix = "s".repeat(2048); + control.emitData(`${noise}${readyLine()}${suffix}`); + expect((await gate.ready).ok).toBe(true); + // The gate drops the pre-READY buffer, so the process no longer retains the + // noise prefix. Without this, the process holds the full sandbox string while + // the ledger charges only the suffix, so retention passes the ceiling. + expect(gate.retainedReadinessBufferLength()).toBe(0); + // The ledger charges only the retained suffix, not the dropped prefix. + expect(ledger.bytesInUse).toBe(Buffer.byteLength(suffix, "utf8")); + expect(ledger.liveTokenCount).toBe(1); + // The broker binds, replays the suffix, and the gate releases the token. + const replayed: string[] = []; + gate.brokerChannel.onData((chunk) => replayed.push(chunk)); + expect(replayed).toEqual([suffix]); + expect(ledger.bytesInUse).toBe(0); + expect(gate.retainedReadinessBufferLength()).toBe(0); + expect(counts.underflows).toBe(0); + }); +}); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index faece1d9bf0..c21b7d3a874 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { Readable } from "node:stream"; import { randomBytes, randomUUID } from "node:crypto"; import type { SshRemoteExecutionSpec } from "./ssh.js"; import { @@ -29,6 +30,7 @@ import { createSandboxCallbackBridgeAsset, createSandboxCallbackBridgeToken, DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES, + DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES, SANDBOX_CALLBACK_BRIDGE_DUPLEX_MODE, SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT, sandboxCallbackBridgeDirectories, @@ -58,11 +60,17 @@ import { DEFAULT_MAX_DUPLEX_FRAME_BYTES, type DuplexRequestFrame, } from "./duplex-frame-codec.js"; +import type { ReassembledBody } from "./duplex-body-spool.js"; import { createDuplexTelemetry, type DuplexFallbackReason, type DuplexTelemetryRecorder, } from "./duplex-telemetry.js"; +import { + DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED, + type DuplexAggregateByteLedger, + type ReservationToken, +} from "./duplex-aggregate-byte-ledger.js"; import { createSshCommandManagedRuntimeRunner, parseSshRemoteExecutionSpec, runSshCommand, shellQuote } from "./ssh.js"; import { ensureCommandResolvable, @@ -184,6 +192,16 @@ export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWor * observability surface. Absent means the safe no-op default. */ duplexTelemetryRecorder?: DuplexTelemetryRecorder | null; + /** + * The process-owned aggregate byte ledger for the sandbox duplex channel. The + * host stamps this same object on every sandbox target on the same seam as + * `runner`, so one shared gauge bounds the aggregate retained bytes across all + * live duplex routes. The live object stays on the host and never enters the + * sandbox environment. The bridge passes it to the broker, the decoder, and the + * response-body reader. Absent means no host ledger; a non-duplex run keeps the + * bridge inert for this seam. + */ + duplexAggregateByteLedger?: DuplexAggregateByteLedger | null; } export type AdapterExecutionTarget = @@ -428,6 +446,20 @@ export function adapterExecutionTargetDuplexTelemetryRecorder( : null; } +/** + * Read the injected aggregate byte ledger off a target. Only a sandbox target + * with a ledger attached returns it. Every other target returns null, so the + * bridge stays inert for this seam. The reader never makes a fresh ledger, so a + * host duplex run always uses the one process-owned ledger the host stamped. + */ +export function adapterExecutionTargetDuplexAggregateByteLedger( + target: AdapterExecutionTarget | null | undefined, +): DuplexAggregateByteLedger | null { + return target?.kind === "remote" && target.transport === "sandbox" + ? target.duplexAggregateByteLedger ?? null + : null; +} + export function adapterExecutionTargetRemoteCwd( target: AdapterExecutionTarget | null | undefined, localCwd: string, @@ -1468,7 +1500,27 @@ function bridgeResponseBodyLimitError(maxBodyBytes: number): Error { return new Error(`Bridge response body exceeded the configured size limit of ${maxBodyBytes} bytes.`); } -async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: number): Promise { +/** + * Read the forward response body into a string. The reader bounds the body with + * two controls. The per-request `maxBodyBytes` limit rejects a body larger than + * the configured per-request ceiling. The optional host aggregate byte ledger + * bounds the retained bytes across all live routes. + * + * The reader charges the ledger for every retained buffer before it allocates + * that buffer. It reserves the exact chunk bytes before it copies a chunk into a + * retained `Buffer`. It reserves the concatenation buffer before it allocates it. + * A reservation that would pass the aggregate ceiling returns no token; the + * reader retains nothing more, cancels the stream reader, and throws the fixed + * marker {@link DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED}. The `finally` releases + * every token exactly one time, so the reader charges the retained bytes only + * while the raw buffers live and never leaves a token held after it returns or + * throws. + */ +async function readBridgeForwardResponseBody( + response: Response, + maxBodyBytes: number, + ledger?: DuplexAggregateByteLedger | null, +): Promise { const rawContentLength = response.headers.get("content-length"); if (rawContentLength) { const contentLength = Number.parseInt(rawContentLength, 10); @@ -1483,19 +1535,54 @@ async function readBridgeForwardResponseBody(response: Response, maxBodyBytes: n const reader = response.body.getReader(); const chunks: Buffer[] = []; + // Every response-body reservation token the reader holds. The `finally` block + // releases each token one time, so a return, a size error, an aggregate + // rejection, and a read error all release every token. + const tokens: ReservationToken[] = []; let totalBytes = 0; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - if (!value) continue; - totalBytes += value.byteLength; - if (totalBytes > maxBodyBytes) { - await reader.cancel().catch(() => undefined); - throw bridgeResponseBodyLimitError(maxBodyBytes); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + const chunkBytes = value.byteLength; + totalBytes += chunkBytes; + if (totalBytes > maxBodyBytes) { + await reader.cancel().catch(() => undefined); + throw bridgeResponseBodyLimitError(maxBodyBytes); + } + // Reserve the exact chunk bytes before the host copies the chunk into a + // retained buffer. A rejection fails closed: cancel the stream reader and + // report the fixed marker; the reader retains nothing more. + if (ledger) { + const token = ledger.reserve("response_body", chunkBytes); + if (!token) { + await reader.cancel().catch(() => undefined); + throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + } + tokens.push(token); + } + chunks.push(Buffer.from(value)); + } + // Reserve the concatenation buffer before the reader allocates it. The + // concatenation buffer is a second copy of the body bytes that lives next to + // the chunk buffers during the concatenation, so it is the peak retained + // allocation. A rejection fails closed with the fixed marker. + if (ledger && totalBytes > 0) { + const concatToken = ledger.reserve("response_body", totalBytes); + if (!concatToken) { + throw new Error(DUPLEX_CHANNEL_AGGREGATE_BYTES_EXCEEDED); + } + tokens.push(concatToken); + } + return Buffer.concat(chunks, totalBytes).toString("utf8"); + } finally { + if (ledger) { + for (const token of tokens) { + ledger.release(token); + } } - chunks.push(Buffer.from(value)); } - return Buffer.concat(chunks, totalBytes).toString("utf8"); } const PROCESS_SESSION_PROXY_SCRIPT = "paperclip-process-session-proxy.mjs"; @@ -2394,7 +2481,8 @@ type DuplexReadinessFailure = | "protocol_contamination" | "nonce_mismatch" | "channel_exit" - | "timeout"; + | "timeout" + | "aggregate_bytes_exceeded"; /** The outcome of the duplex readiness handshake. */ type DuplexReadinessResult = @@ -2416,6 +2504,8 @@ function duplexReadinessFallbackReason(reason: DuplexReadinessFailure): DuplexFa return "ready_timeout"; case "channel_exit": return "ready_invalid"; + case "aggregate_bytes_exceeded": + return "aggregate_bytes_exceeded"; } } @@ -2542,6 +2632,13 @@ export const __duplexReadinessTesting = { resetNewlineScanUnits: (): void => { duplexReadinessNewlineScanUnits = 0; }, + // Build one readiness gate over a supplied channel, so a test can drive the + // readiness-replay reservation lifecycle across every terminal path without the + // whole bridge. Production code never reads this factory. + createReadinessGate: ( + channel: CommandManagedDuplexChannel, + options: { nonce: string; timeoutMs: number; ledger?: DuplexAggregateByteLedger | null }, + ) => createDuplexReadinessGate(channel, options), }; interface DuplexReadinessGate { @@ -2554,14 +2651,82 @@ interface DuplexReadinessGate { * the channel. */ readonly brokerChannel: CommandManagedDuplexChannel; + /** + * Report whether a post-READY pre-bind chunk could not reserve its replay bytes + * against the aggregate ledger. On such a refusal the gate drops the pending + * replay buffer and stops the channel. The caller reads this after `ready` + * resolves `ok`, and before it binds the broker. A `true` result means the caller + * must abandon the broker and select the file bridge with the aggregate marker. + */ + replayOverflowed(): boolean; + /** + * Release every held readiness-replay reservation exactly once and drop the + * pending replay buffer. The caller runs this on a terminal path that abandons + * the pending replay without a broker handoff: a readiness failure, a replay + * overflow, or a broker-construction failure. The normal handoff releases the + * reservation inside `brokerChannel.onData`, so a later call here is a no-op. + */ + disposePendingReplay(): void; + /** + * Test-only. Report the length of the retained pre-READY buffer, in UTF-16 code + * units. A test reads this to prove the gate drops the pre-READY buffer on READY + * acceptance, so the process does not retain the sandbox-controlled prefix. + * Production code does not read this. + */ + retainedReadinessBufferLength(): number; } function createDuplexReadinessGate( channel: CommandManagedDuplexChannel, - options: { nonce: string; timeoutMs: number }, + options: { + nonce: string; + timeoutMs: number; + // The one host-process aggregate byte ledger. The gate charges the untrusted + // pre-READY buffer bytes against it, so a pre-READY flood counts toward the + // aggregate ceiling across all live routes. A gate with no ledger stays inert + // for this seam. The gate holds the same object every other host retention + // site holds, so the aggregate identity holds at this seam. + ledger?: DuplexAggregateByteLedger | null; + }, ): DuplexReadinessGate { + const ledger = options.ledger ?? null; let settled = false; let readyOk = false; + // Every readiness-buffer reservation token the gate holds for the pre-READY + // bytes. The gate releases each token one time when it settles or when it + // accepts READY. On a failed handshake the gate drops the buffer, so the release + // frees the untrusted bytes. On READY the gate discards the whole pre-READY + // buffer, then re-charges only the retained suffix under `readiness_replay`. + const retainedTokens: ReservationToken[] = []; + // Every readiness-replay reservation token the gate holds for the post-READY + // suffix and each later pre-bind chunk. The gate releases each token one time + // after the synchronous handoff to the broker, or on a terminal path that + // abandons the pending replay without a broker handoff. + const replayTokens: ReservationToken[] = []; + // The gate sets this when a post-READY pre-bind chunk cannot reserve its replay + // bytes. On that refusal the gate drops the pending buffer and stops the channel. + // The caller reads it through `replayOverflowed` and selects the file bridge. + let replayOverflow = false; + + // Release every readiness-buffer token exactly once and clear the registry. A + // second call is a no-op, because the array is empty. + function releaseReadinessBufferTokens(): void { + if (!ledger) return; + for (const token of retainedTokens) { + ledger.release(token); + } + retainedTokens.length = 0; + } + + // Release every readiness-replay token exactly once and clear the registry. A + // second call is a no-op, because the array is empty. + function releaseReplayTokens(): void { + if (!ledger) return; + for (const token of replayTokens) { + ledger.release(token); + } + replayTokens.length = 0; + } // The raw bytes the host reads before the READY frame completes. The buffer is // append-only, so the O(1) cap check on `buffer.length` stays valid. let buffer = ""; @@ -2590,6 +2755,11 @@ function createDuplexReadinessGate( settled = true; clearTimeout(timer); if (result.ok) readyOk = true; + // Release every readiness-buffer token exactly once. The gate no longer owns + // the pre-READY bytes: a failed handshake drops the buffer. The READY-accept + // path already released these tokens and charged the retained suffix under + // `readiness_replay`, so this call is a no-op there. + releaseReadinessBufferTokens(); resolveReady(result); } @@ -2599,7 +2769,24 @@ function createDuplexReadinessGate( return; } if (readyOk) { - // READY already passed; hold the bytes until the broker binds. + // READY already passed; hold the bytes until the broker binds. Reserve the + // exact UTF-8 bytes under `readiness_replay` before the append, so the replay + // buffer counts toward the aggregate ceiling. A refusal fails closed: the gate + // drops the pending buffer, releases the replay tokens, stops the channel, and + // sets the overflow flag. The caller reads the flag and selects the file bridge + // with the aggregate marker, because `ready` already resolved before this + // synchronous post-READY chunk arrived. + if (ledger) { + const token = ledger.reserve("readiness_replay", Buffer.byteLength(chunk, "utf8")); + if (!token) { + replayOverflow = true; + pending = ""; + releaseReplayTokens(); + channel.stop(); + return; + } + replayTokens.push(token); + } pending += chunk; return; } @@ -2609,6 +2796,18 @@ function createDuplexReadinessGate( // never grows the buffer after the gate settles. return; } + // Reserve the exact UTF-8 bytes of this chunk against the aggregate ledger + // before the gate retains it. The pre-READY buffer holds untrusted bytes, so + // a flood counts toward the process aggregate ceiling. A rejection fails + // closed: the gate retains nothing more and falls back to the file bridge. + if (ledger) { + const token = ledger.reserve("readiness_buffer", Buffer.byteLength(chunk, "utf8")); + if (!token) { + finish({ ok: false, reason: "aggregate_bytes_exceeded" }); + return; + } + retainedTokens.push(token); + } // Append the new bytes and continue the newline search from `scanFrom`, the // first index not yet examined. Each code unit is read at most one time for // the search, so the total scan work stays linear in the bytes received. @@ -2682,8 +2881,33 @@ function createDuplexReadinessGate( finish({ ok: false, reason: "nonce_mismatch" }); return; } - // Hold the bytes that follow the READY line for the broker to replay. - pending = buffer.slice(newlineIndex + 1); + // The bytes that follow the READY line become the replay buffer for the + // broker. Drop the whole pre-READY buffer charge first, then reserve the + // retained suffix under `readiness_replay`. The release-before-reserve order + // keeps the transient charge equal to the suffix, not the sum of the dropped + // prefix and the retained suffix. The two steps run in one synchronous + // section, so no other route can take the freed bytes in between. + const suffix = buffer.slice(newlineIndex + 1); + // Drop the original pre-READY buffer now. The gate keeps only the + // retained suffix as `pending`, and it charges that suffix under + // `readiness_replay` below. If the gate keeps the buffer, the process + // retains the full sandbox-controlled string while the ledger counts + // only the suffix, so aggregate retention passes the ceiling. This clear + // also covers the broker handoff and the replay disposal. Both run later + // and read no buffer bytes. + buffer = ""; + releaseReadinessBufferTokens(); + if (ledger && suffix.length > 0) { + const token = ledger.reserve("readiness_replay", Buffer.byteLength(suffix, "utf8")); + if (!token) { + // The retained suffix passes the aggregate ceiling. Fail closed: drop + // the suffix and fall back to the file bridge with the aggregate marker. + finish({ ok: false, reason: "aggregate_bytes_exceeded" }); + return; + } + replayTokens.push(token); + } + pending = suffix; finish({ ok: true }); return; } @@ -2723,6 +2947,11 @@ function createDuplexReadinessGate( pending = ""; listener(replay); } + // Release every readiness-replay token exactly once, after the synchronous + // handoff to the broker. The broker charges its own decode retention inside + // the `listener(replay)` call above, so the release here never opens an + // admission gap for the same retained bytes. + releaseReplayTokens(); }, onExit: (listener: (exit: { exitCode: number | null }) => void) => { exitSink = listener; @@ -2736,7 +2965,16 @@ function createDuplexReadinessGate( close: () => channel.close(), }; - return { ready, brokerChannel }; + return { + ready, + brokerChannel, + replayOverflowed: () => replayOverflow, + disposePendingReplay: () => { + pending = ""; + releaseReplayTokens(); + }, + retainedReadinessBufferLength: () => buffer.length, + }; } /** @@ -2817,6 +3055,12 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { } const target = input.target; + // The process-owned aggregate byte ledger the host stamped on this sandbox + // target. The forward response-body reader charges its retained bytes against + // this one ledger, so the aggregate retained bytes across all live routes stay + // under the ceiling. A target with no ledger keeps the reader inert for this + // seam. + const duplexAggregateByteLedger = adapterExecutionTargetDuplexAggregateByteLedger(target); const onLog = input.onLog ?? (async () => {}); const hostApiToken = input.hostApiToken?.trim() ?? ""; if (hostApiToken.length === 0) { @@ -2890,9 +3134,24 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // file bridge and the duplex broker. The sandbox request carries only the // bridge token; the real agent token never leaves the host. const forwardBridgeRequest = async ( - request: { method: string; path: string; query: string; headers: Record; body: string }, + request: { + method: string; + path: string; + query: string; + headers: Record; + /** The file bridge passes the whole request body here as one string. */ + body?: string; + }, signal?: AbortSignal, - options?: { suppressDebugLog?: boolean }, + options?: { + suppressDebugLog?: boolean; + /** + * The duplex broker passes the reassembled request body here. The forward + * streams it to the host, so a spilled body never loads into memory on the + * receive side. The forward never disposes it; the broker owns its lifecycle. + */ + reassembledBody?: ReassembledBody; + }, ): Promise<{ status: number; headers: Record; body: string }> => { const method = request.method.trim().toUpperCase() || "GET"; // The per-request debug log prints the method, the path, and the query. The @@ -2917,12 +3176,26 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // the forward budget here, whichever comes first. const timeoutSignal = AbortSignal.timeout(forwardTimeoutMs); const forwardSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; - const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), { + // Build the request-body init. A GET or a HEAD carries no body. The duplex + // broker passes the reassembled body: stream it, so a body that spilled to + // disk never loads into memory here. A streamed body needs `duplex: "half"`. + // The file bridge passes the whole body as one string. + const forwardInit: RequestInit & { duplex?: "half" } = { method, headers, - ...(method === "GET" || method === "HEAD" ? {} : { body: request.body }), signal: forwardSignal, - }); + }; + if (method !== "GET" && method !== "HEAD") { + if (options?.reassembledBody) { + forwardInit.body = Readable.toWeb( + options.reassembledBody.createReadStream(), + ) as unknown as ReadableStream; + forwardInit.duplex = "half"; + } else if (typeof request.body === "string") { + forwardInit.body = request.body; + } + } + const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), forwardInit); if (emitDebugLog) { await onLog( "stdout", @@ -2943,7 +3216,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // to a non-retryable 409 for both the file bridge and the duplex broker. let responseBody: string; try { - responseBody = await readBridgeForwardResponseBody(response, maxBodyBytes); + responseBody = await readBridgeForwardResponseBody( + response, + maxBodyBytes, + duplexAggregateByteLedger, + ); } catch (error) { if (isSafeBridgeMethod(method)) { // The method is safe, so a retry cannot double-apply a mutation. Return a @@ -3049,6 +3326,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { PAPERCLIP_BRIDGE_PORT: String(assignedPort), PAPERCLIP_BRIDGE_NONCE: nonce, PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes), + // The separate sandbox-process raw-decoder cap. The generated gateway runs + // in a different operating-system process, so it cannot share the host + // aggregate byte ledger. It enforces this cap locally under the + // `sandbox_process` scope, and the provider memory allocation bounds it. + PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES: String(DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES), }; const command = buildDuplexGatewayLaunchArgv({ shellCommand, @@ -3080,7 +3362,14 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { } if (channel) { - const gate = createDuplexReadinessGate(channel, { nonce, timeoutMs: readinessTimeoutMs }); + const gate = createDuplexReadinessGate(channel, { + nonce, + timeoutMs: readinessTimeoutMs, + // Inject the one host-process aggregate byte ledger, the same object the + // broker and the response-body reader hold. The gate charges the untrusted + // pre-READY buffer against it, so the aggregate identity holds at this seam. + ledger: duplexAggregateByteLedger, + }); const readiness = await gate.ready; if (!readiness.ok) { // Fail closed. Close the partial channel inside a bounded budget, then @@ -3088,12 +3377,26 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // carries the bridge token reached the channel or any endpoint. The reason // is a fixed enum, so it rides the log line and the fallback telemetry // with no raw value. + gate.disposePendingReplay(); await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); duplexChannelOpen.fallback(duplexReadinessFallbackReason(readiness.reason)); await onLog( "stderr", `[paperclip] Sandbox duplex readiness failed (${readiness.reason}). Using the file bridge.\n`, ); + } else if (gate.replayOverflowed()) { + // Readiness passed, but a post-READY pre-bind chunk passed the aggregate + // byte ceiling. The gate dropped the replay buffer and stopped the channel. + // Release any held replay reservation, close the partial channel inside the + // cleanup budget, and select the file bridge with the aggregate marker. The + // broker never bound, so no request reached the channel or any endpoint. + gate.disposePendingReplay(); + await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); + duplexChannelOpen.fallback(duplexReadinessFallbackReason("aggregate_bytes_exceeded")); + await onLog( + "stderr", + "[paperclip] Sandbox duplex readiness replay exceeded the aggregate byte ceiling (aggregate_bytes_exceeded). Using the file bridge.\n", + ); } else { // Readiness passed. Construct the broker inside the guarded region, so a // construction throw closes the channel within the cleanup budget and @@ -3103,14 +3406,14 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { // token only now, after readiness passed. let broker: DuplexBridgeBroker | null = null; try { - broker = createDuplexBridgeBroker({ + broker = await createDuplexBridgeBroker({ channel: gate.brokerChannel, // Derive the nested budgets from the forward budget, so any forward // budget keeps the nested order the broker asserts at construction. budgets: deriveNestedDuplexBrokerBudgets(forwardTimeoutMs), forwardRequest: async ( request: DuplexRequestFrame, - options: { signal: AbortSignal }, + options: { signal: AbortSignal; body: ReassembledBody }, ): Promise => { const denialReason = authorizeSandboxCallbackBridgeRequestWithRoutes(request); if (denialReason) { @@ -3130,9 +3433,11 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { headers: sanitizeSandboxCallbackBridgeHeaders(request.headers), }; // Suppress the per-request debug log on the duplex path, so no route - // or query rides a log line here. + // or query rides a log line here. Stream the reassembled request + // body, so a spilled body never loads into memory on the host. return forwardBridgeRequest(sanitizedRequest, options.signal, { suppressDebugLog: true, + reassembledBody: options.body, }); }, // The duplex path emits only the fixed transport telemetry. It passes no @@ -3149,12 +3454,20 @@ export async function startAdapterExecutionTargetPaperclipBridge(input: { `[paperclip] Sandbox duplex channel lost (${typedDuplexLossReason(record.reason)}). The run fails.\n`, ); }, + // Inject the one host-process aggregate byte ledger. The broker + // reserves the retained request-frame, request-payload, and no-replay + // set-entry bytes against it, so the aggregate retained bytes across all + // live routes stay under the ceiling. It is the same object the + // response-body reader reads off the stamped target above. + duplexAggregateByteLedger, }); } catch { - // The broker construction failed, so no broker owns the channel. Close - // the channel within the cleanup budget, then select the file bridge. - // The log line names no raw error, so no raw error rides a log line on - // the duplex path. + // The broker construction failed, so no broker owns the channel. The + // broker never bound, so it never released the pending replay reservation; + // release it here. Then close the channel within the cleanup budget and + // select the file bridge. The log line names no raw error, so no raw error + // rides a log line on the duplex path. + gate.disposePendingReplay(); await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); duplexChannelOpen.fallback("broker_construction_failed"); await onLog( diff --git a/packages/adapter-utils/src/index.ts b/packages/adapter-utils/src/index.ts index aa7bb230299..283097815d4 100644 --- a/packages/adapter-utils/src/index.ts +++ b/packages/adapter-utils/src/index.ts @@ -86,7 +86,6 @@ export type { export { inferOpenAiCompatibleBiller } from "./billing.js"; export { ADAPTER_LOGIN_PANEL_MODES, - ADAPTER_LOGIN_SANDBOX_TRANSPORTS, ADAPTER_LOGIN_TIMEOUT_POLICIES, ADAPTER_LOGIN_COMPLETION_CLAIMS, assertValidAdapterLoginCapability, @@ -94,7 +93,6 @@ export { } from "./login-capability.js"; export type { AdapterLoginPanelMode, - AdapterLoginSandboxTransport, AdapterLoginTimeoutPolicy, AdapterLoginCompletionClaim, AdapterLoginPrompt, diff --git a/packages/adapter-utils/src/login-capability.test.ts b/packages/adapter-utils/src/login-capability.test.ts index a79a4e2dfa1..617103defce 100644 --- a/packages/adapter-utils/src/login-capability.test.ts +++ b/packages/adapter-utils/src/login-capability.test.ts @@ -10,7 +10,6 @@ import { function validCapability(): AdapterLoginCapability { return { panelMode: "displayed_code", - sandboxTransport: "streamed_exec", timeoutPolicy: "caller_bounded", getCommand: () => "vendor login", parsePrompt: () => null, @@ -25,7 +24,6 @@ describe("assertValidAdapterLoginCapability", () => { it("accepts a well-formed capability with every optional member set", () => { const capability: AdapterLoginCapability = { panelMode: "submitted_browser_code", - sandboxTransport: "pseudo_terminal", timeoutPolicy: "fixed", getCommand: () => "vendor setup-token", parsePrompt: () => null, @@ -48,11 +46,6 @@ describe("assertValidAdapterLoginCapability", () => { expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/panelMode/); }); - it("rejects an unknown sandboxTransport", () => { - const bad = { ...validCapability(), sandboxTransport: "http" }; - expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/sandboxTransport/); - }); - it("rejects an unknown timeoutPolicy", () => { const bad = { ...validCapability(), timeoutPolicy: "unbounded" }; expect(() => assertValidAdapterLoginCapability(bad, "vendor")).toThrow(/timeoutPolicy/); diff --git a/packages/adapter-utils/src/login-capability.ts b/packages/adapter-utils/src/login-capability.ts index 986dc8a6b6e..9eac8e13466 100644 --- a/packages/adapter-utils/src/login-capability.ts +++ b/packages/adapter-utils/src/login-capability.ts @@ -18,14 +18,6 @@ export const ADAPTER_LOGIN_PANEL_MODES = ["displayed_code", "submitted_browser_code"] as const; export type AdapterLoginPanelMode = (typeof ADAPTER_LOGIN_PANEL_MODES)[number]; -/** - * The sandbox transport that the login command needs. `streamed_exec` runs the - * command over the streamed exec channel. `pseudo_terminal` runs the command on - * a real pseudo-terminal, because a pipe emits no login prompt. - */ -export const ADAPTER_LOGIN_SANDBOX_TRANSPORTS = ["streamed_exec", "pseudo_terminal"] as const; -export type AdapterLoginSandboxTransport = (typeof ADAPTER_LOGIN_SANDBOX_TRANSPORTS)[number]; - /** * The host-side timeout policy. `caller_bounded` lets the caller set the * timeout. `fixed` binds the timeout to a fixed adapter value. @@ -68,8 +60,6 @@ export interface AdapterLoginCompletionContext { export interface AdapterLoginCapability { /** The login panel mode. */ panelMode: AdapterLoginPanelMode; - /** The sandbox transport that the login command needs. */ - sandboxTransport: AdapterLoginSandboxTransport; /** The host-side timeout policy. */ timeoutPolicy: AdapterLoginTimeoutPolicy; /** Returns the fixed, non-secret login command. */ @@ -122,11 +112,6 @@ export function assertValidAdapterLoginCapability( `${prefix}: "panelMode" must be one of ${ADAPTER_LOGIN_PANEL_MODES.join(", ")}.`, ); } - if (!isOneOf(ADAPTER_LOGIN_SANDBOX_TRANSPORTS, cap.sandboxTransport)) { - throw new Error( - `${prefix}: "sandboxTransport" must be one of ${ADAPTER_LOGIN_SANDBOX_TRANSPORTS.join(", ")}.`, - ); - } if (!isOneOf(ADAPTER_LOGIN_TIMEOUT_POLICIES, cap.timeoutPolicy)) { throw new Error( `${prefix}: "timeoutPolicy" must be one of ${ADAPTER_LOGIN_TIMEOUT_POLICIES.join(", ")}.`, diff --git a/packages/adapter-utils/src/setup-token-transport.test.ts b/packages/adapter-utils/src/login-pty-transport.test.ts similarity index 88% rename from packages/adapter-utils/src/setup-token-transport.test.ts rename to packages/adapter-utils/src/login-pty-transport.test.ts index 3644e35581e..d73bddcb166 100644 --- a/packages/adapter-utils/src/setup-token-transport.test.ts +++ b/packages/adapter-utils/src/login-pty-transport.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "vitest"; import { - createSetupTokenPtyTransport, - type SetupTokenPtySession, -} from "./setup-token-transport.js"; + createLoginPtyTransport, + type LoginPtySession, +} from "./login-pty-transport.js"; // The Enter byte the terminal login UI reads to submit the browser code. The // login runner appends this byte to the code. The transport forwards the bytes @@ -14,7 +14,7 @@ const ENTER = "\r"; * stream on demand, and records the stop and the close. The tests use it in * place of a real pseudo-terminal, so the transport runs with no sandbox. */ -function createFakePtySession(): SetupTokenPtySession & { +function createFakePtySession(): LoginPtySession & { writes: string[]; emit: (chunk: string) => void; finish: (exitCode: number | null) => void; @@ -74,10 +74,10 @@ function createFakePtySession(): SetupTokenPtySession & { }; } -describe("createSetupTokenPtyTransport", () => { +describe("createLoginPtyTransport", () => { it("delivers delayed input to the process", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); await session.ready; @@ -94,7 +94,7 @@ describe("createSetupTokenPtyTransport", () => { it("returns incremental terminal output to the runner", async () => { const session = createFakePtySession(); const received: string[] = []; - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", (chunk) => { received.push(chunk); @@ -114,7 +114,7 @@ describe("createSetupTokenPtyTransport", () => { it("delivers the Enter byte after the browser code", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); await session.ready; @@ -133,7 +133,7 @@ describe("createSetupTokenPtyTransport", () => { it("resolves start with the child exit code", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); session.finish(7); @@ -143,7 +143,7 @@ describe("createSetupTokenPtyTransport", () => { it("stops the child with a direct kill", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); await session.ready; @@ -156,7 +156,7 @@ describe("createSetupTokenPtyTransport", () => { it("disposes the session resources", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); const started = transport.start("claude setup-token", () => {}); session.finish(0); @@ -168,7 +168,7 @@ describe("createSetupTokenPtyTransport", () => { it("stays safe when stop and dispose run before start", async () => { const session = createFakePtySession(); - const transport = createSetupTokenPtyTransport(async () => session); + const transport = createLoginPtyTransport(async () => session); // The runner may stop or dispose before it starts the child. The transport // must not throw, and it must not open a session for a run it already stopped. @@ -185,7 +185,7 @@ describe("createSetupTokenPtyTransport", () => { const openGate = new Promise((resolve) => { resolveOpen = resolve; }); - const transport = createSetupTokenPtyTransport(async () => { + const transport = createLoginPtyTransport(async () => { await openGate; return session; }); @@ -207,7 +207,7 @@ describe("createSetupTokenPtyTransport", () => { const openGate = new Promise((resolve) => { resolveOpen = resolve; }); - const transport = createSetupTokenPtyTransport(async () => { + const transport = createLoginPtyTransport(async () => { await openGate; return session; }); diff --git a/packages/adapter-utils/src/setup-token-transport.ts b/packages/adapter-utils/src/login-pty-transport.ts similarity index 89% rename from packages/adapter-utils/src/setup-token-transport.ts rename to packages/adapter-utils/src/login-pty-transport.ts index af8b0c856cc..48369bdbe49 100644 --- a/packages/adapter-utils/src/setup-token-transport.ts +++ b/packages/adapter-utils/src/login-pty-transport.ts @@ -1,4 +1,4 @@ -// The setup-token pseudo-terminal transport. It gives the Claude `setup-token` +// The login pseudo-terminal (PTY) transport. It gives the Claude `setup-token` // login runner a child that runs on a real pseudo-terminal (PTY). The command // needs a PTY: pipe stdio emits no login prompt. The transport starts the // command on a PTY, streams the incremental terminal output, delivers delayed @@ -7,7 +7,7 @@ // This module is provider-agnostic and pure. It holds no Node built-in import, // so the browser-safe root entry can re-export it. A sandbox provider (for // example the Daytona plugin) opens the concrete pseudo-terminal through a -// {@link SetupTokenPtySessionOpener}. A unit test opens a fake pseudo-terminal +// {@link LoginPtySessionOpener}. A unit test opens a fake pseudo-terminal // through the same opener. // // Boundary (ANSI and OSC 8): the transport forwards the raw terminal bytes @@ -20,7 +20,7 @@ * raw terminal output, accepts delayed input, and stops the child. The session * forwards the raw terminal bytes; it runs no ANSI or OSC 8 handling. */ -export interface SetupTokenPtySession { +export interface LoginPtySession { /** * Registers the one output listener. The session streams each raw terminal * output chunk to `listener`, in order, as the pseudo-terminal emits it. @@ -49,7 +49,7 @@ export interface SetupTokenPtySession { * Opens a pseudo-terminal session for `command`. A provider binds this to its * sandbox. The transport calls it one time, on start. */ -export type SetupTokenPtySessionOpener = (command: string) => Promise; +export type LoginPtySessionOpener = (command: string) => Promise; /** * The child side of the setup-token run, in the shape the login runner needs. @@ -60,7 +60,7 @@ export type SetupTokenPtySessionOpener = (command: string) => Promise { if (started) { - throw new Error("setup-token PTY transport already started."); + throw new Error("login PTY transport already started."); } started = true; const opened = await open(command); diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index fc5601708aa..5d8907ce048 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -19,6 +19,14 @@ const DEFAULT_BRIDGE_RESPONSE_TIMEOUT_MS = 30_000; const DEFAULT_BRIDGE_STOP_TIMEOUT_MS = 2_000; const DEFAULT_BRIDGE_MAX_QUEUE_DEPTH = 64; const DEFAULT_BRIDGE_MAX_BODY_BYTES = 256 * 1024; +// The default cap on the aggregate raw bytes the in-sandbox duplex frame decoder +// retains between chunks. The generated gateway runs in a separate operating-system +// process, so it cannot share the host aggregate byte ledger. It enforces this +// separate cap locally under the `sandbox_process` scope, and the provider memory +// allocation bounds it. The host passes the value through +// PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES. The default is well above one maximum +// frame, so a legitimate single frame never trips it. +const DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES = 8 * 1024 * 1024; // Per-iteration timeout for one poll-loop client call. A healthy control-plane // round trip finishes in well under one second, so 10s is far above a normal // iteration and never false-fires on a slow-but-live call. It is also well @@ -89,6 +97,18 @@ const CALLBACK_BRIDGE_WORKER_FAILED_SPAN = "sandbox.callbackBridge.workerFailed" export const DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES = DEFAULT_BRIDGE_MAX_BODY_BYTES; +/** + * The default cap on the aggregate raw bytes the generated in-sandbox duplex frame + * decoder retains. The host passes it through + * `PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES`. The in-sandbox decoder enforces the + * cap locally under the `sandbox_process` scope; it never touches the host + * aggregate byte ledger. + */ +export const DEFAULT_SANDBOX_DUPLEX_DECODER_MAX_BYTES = DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES; + +/** The scope label the in-sandbox duplex decoder reports for its local byte counter. */ +export const SANDBOX_DUPLEX_DECODER_SCOPE = "sandbox_process"; + export interface SandboxCallbackBridgeRouteRule { method: string; path: RegExp; @@ -1743,9 +1763,12 @@ export async function startSandboxCallbackBridgeServer(input: { * compatible with the host codec. {@link getSandboxDuplexGatewayCodecSource} * returns this exact source so a test can run every fixture vector against it. */ -const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 1; +const DUPLEX_GATEWAY_CODEC_SOURCE = `const DUPLEX_FRAME_VERSION = 2; +const DUPLEX_BODY_CHUNK_RAW_BYTES = 256 * 1024; const DEFAULT_MAX_DUPLEX_FRAME_BYTES = 1000000; const DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES = 256; +const DEFAULT_MAX_DUPLEX_DECODER_BYTES = ${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}; +const DUPLEX_DECODER_SCOPE = "${SANDBOX_DUPLEX_DECODER_SCOPE}"; const DUPLEX_NEWLINE_BYTE = 0x0a; const DUPLEX_EMPTY = Buffer.alloc(0); const DUPLEX_RESPONSE_OUTCOMES = new Set(["completed", "indeterminate", "unavailable"]); @@ -1770,10 +1793,23 @@ function duplexIsStringRecord(value) { return true; } +function duplexIsSafeNonNegativeInteger(value) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + function encodeDuplexFrame(frame) { return JSON.stringify(frame) + "\\n"; } +function encodeDuplexFrameChecked(frame, maxFrameBytes) { + const limit = maxFrameBytes != null ? maxFrameBytes : DEFAULT_MAX_DUPLEX_FRAME_BYTES; + const json = JSON.stringify(frame); + if (Buffer.byteLength(json, "utf8") > limit) { + return { ok: false, error: { code: "frame_too_large", message: "frame exceeds the maximum size" } }; + } + return { ok: true, line: json + "\\n" }; +} + function decodeDuplexLine(line) { const text = typeof line === "string" ? line : line.toString("utf8"); let parsed; @@ -1797,6 +1833,8 @@ function duplexValidateFrame(frame) { return duplexValidateRequest(frame); case "response": return duplexValidateResponse(frame); + case "body_chunk": + return duplexValidateBodyChunk(frame); case "ready": return duplexValidateReady(frame); case "heartbeat": @@ -1816,7 +1854,7 @@ function duplexValidateRequest(frame) { typeof frame.method !== "string" || typeof frame.path !== "string" || typeof frame.query !== "string" || - typeof frame.body !== "string" || + !duplexIsSafeNonNegativeInteger(frame.bodyByteCount) || !duplexIsStringRecord(frame.headers) ) { return duplexFail("malformed_frame", "request frame has a missing or wrong-typed field"); @@ -1831,7 +1869,7 @@ function duplexValidateResponse(frame) { if ( typeof frame.id !== "string" || typeof frame.status !== "number" || - typeof frame.body !== "string" || + !duplexIsSafeNonNegativeInteger(frame.bodyByteCount) || !duplexIsStringRecord(frame.headers) || typeof frame.outcome !== "string" || !DUPLEX_RESPONSE_OUTCOMES.has(frame.outcome) @@ -1844,6 +1882,19 @@ function duplexValidateResponse(frame) { return duplexOk(frame); } +function duplexValidateBodyChunk(frame) { + if (typeof frame.id !== "string" || typeof frame.data !== "string") { + return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed field"); + } + if (!duplexIsSafeNonNegativeInteger(frame.seq)) { + return duplexFail("malformed_frame", "body_chunk frame has a missing or wrong-typed seq"); + } + if (Buffer.byteLength(frame.id, "utf8") > DEFAULT_MAX_DUPLEX_REQUEST_ID_BYTES) { + return duplexFail("id_too_large", "body_chunk frame id exceeds the maximum size"); + } + return duplexOk(frame); +} + function duplexValidateReady(frame) { if (typeof frame.nonce !== "string") { return duplexFail("malformed_frame", "ready frame has a missing or wrong-typed nonce"); @@ -1872,10 +1923,32 @@ class DuplexFrameDecoder { this.discarding = false; this.maxFrameBytes = options && options.maxFrameBytes != null ? options.maxFrameBytes : DEFAULT_MAX_DUPLEX_FRAME_BYTES; + // The in-sandbox decoder runs in a separate operating-system process, so it + // cannot share the host aggregate byte ledger. It bounds its own retained + // bytes with a local cap under the "sandbox_process" scope. The counters here + // never increment or release a host aggregate token. + this.scope = DUPLEX_DECODER_SCOPE; + this.maxAggregateBytes = + options && options.maxAggregateBytes != null + ? options.maxAggregateBytes + : DEFAULT_MAX_DUPLEX_DECODER_BYTES; + this.bytesInUse = 0; + this.aggregateRejections = 0; } push(chunk) { const incoming = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : Buffer.from(chunk); + // Enforce the local sandbox-process cap before the concat allocates the peak + // "old + incoming" buffer. A rejection fails closed: the decoder drops the + // retained buffer and the incoming chunk, resynchronizes at the next newline, + // and reports the aggregate rejection. It retains nothing over the cap. + if (this.buffer.length + incoming.length > this.maxAggregateBytes) { + this.buffer = DUPLEX_EMPTY; + this.discarding = true; + this.bytesInUse = 0; + this.aggregateRejections += 1; + return [duplexFail("aggregate_bytes_exceeded", "aggregate retained bytes exceeded the sandbox-process cap")]; + } this.buffer = this.buffer.length === 0 ? incoming : Buffer.concat([this.buffer, incoming]); const results = []; for (;;) { @@ -1907,6 +1980,8 @@ class DuplexFrameDecoder { } results.push(decodeDuplexLine(line)); } + // Reconcile the local sandbox-process counter to the bytes still retained. + this.bytesInUse = this.buffer.length; return results; } }`; @@ -1915,9 +1990,10 @@ class DuplexFrameDecoder { * Return the exact zero-dependency codec source the generated duplex gateway * embeds. A test runs every fixture vector against this source, so it proves the * embedded copy decodes the same bytes as the host codec. The source declares - * `encodeDuplexFrame`, `decodeDuplexLine`, `DuplexFrameDecoder`, - * `DUPLEX_FRAME_VERSION`, and `DEFAULT_MAX_DUPLEX_FRAME_BYTES`, but exports none - * of them; a caller wraps it to read those names. + * `encodeDuplexFrame`, `encodeDuplexFrameChecked`, `decodeDuplexLine`, + * `DuplexFrameDecoder`, `DUPLEX_FRAME_VERSION`, and + * `DEFAULT_MAX_DUPLEX_FRAME_BYTES`, but exports none of them; a caller wraps it to + * read those names. */ export function getSandboxDuplexGatewayCodecSource(): string { return DUPLEX_GATEWAY_CODEC_SOURCE; @@ -1949,6 +2025,12 @@ const responseTimeoutMs = Number( ); const maxQueueDepth = Number(process.env.PAPERCLIP_BRIDGE_MAX_QUEUE_DEPTH || "${DEFAULT_BRIDGE_MAX_QUEUE_DEPTH}"); const maxBodyBytes = Number(process.env.PAPERCLIP_BRIDGE_MAX_BODY_BYTES || "${DEFAULT_BRIDGE_MAX_BODY_BYTES}"); +// The host passes the separate sandbox-process raw-decoder cap here. The in-sandbox +// decoder enforces it locally under the "sandbox_process" scope; it never shares the +// host aggregate byte ledger. +const maxDuplexDecoderBytes = Number( + process.env.PAPERCLIP_BRIDGE_MAX_DUPLEX_DECODER_BYTES || "${DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES}", +); const heartbeatIntervalMs = Number( process.env.PAPERCLIP_BRIDGE_HEARTBEAT_INTERVAL_MS || "${DEFAULT_DUPLEX_GATEWAY_HEARTBEAT_INTERVAL_MS}", ); @@ -2136,10 +2218,37 @@ async function runFileGateway() { }); } +// Split one whole body buffer into body_chunk frames. The gateway buffers the +// whole source body once, then splits it here. Each frame carries one fixed raw +// slice as base64 text, except the final frame, which carries the remaining +// bytes. A zero-length body yields no frame. Send-side true streaming is a +// separate goal; this whole-body split is acceptable for this gateway. +function splitDuplexBodyIntoChunks(id, body) { + const frames = []; + let seq = 0; + for (let offset = 0; offset < body.length; offset += DUPLEX_BODY_CHUNK_RAW_BYTES) { + const slice = body.subarray(offset, offset + DUPLEX_BODY_CHUNK_RAW_BYTES); + frames.push({ + version: DUPLEX_FRAME_VERSION, + type: "body_chunk", + id: id, + seq: seq, + data: slice.toString("base64"), + }); + seq += 1; + } + return frames; +} + function runDuplexGateway() { // One outstanding local request per id. Each entry holds the HTTP resolver and // the wait-budget timer. const pending = new Map(); + // One in-flight response reassembly per id. The host returns a response as an + // envelope frame that carries bodyByteCount, then the body_chunk frames that + // carry the body. The gateway reassembles the response body in memory here; it + // does not spill, because a production response body stays small. + const responseAssembly = new Map(); let unavailable = false; let lossTriggered = false; let lastInboundAt = Date.now(); @@ -2181,27 +2290,102 @@ function runDuplexGateway() { }); } pending.clear(); + responseAssembly.clear(); const exitTimer = setTimeout(() => process.exit(0), lossExitGraceMs); if (typeof exitTimer.unref === "function") exitTimer.unref(); } + // Fail one outstanding request with a bounded local 502. The gateway calls it + // when a response reassembly breaks: a reordered seq, a base64 that is not + // canonical, or a total that overruns the declared body size. The host is the + // response peer, so this is a defensive local error, not a channel loss. + function failRequest(id, message) { + responseAssembly.delete(id); + const entry = pending.get(id); + if (!entry) return; + pending.delete(id); + clearTimeout(entry.timer); + entry.resolve({ + status: 502, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ error: message }), + }); + } + function handleInboundFrame(frame) { if (frame.type === "response") { const entry = pending.get(frame.id); if (!entry) return; - pending.delete(frame.id); - clearTimeout(entry.timer); // Map an indeterminate outcome to a non-retryable 409, the same contract // the file gateway applies through the outcome header. const statusCode = frame.outcome === "indeterminate" ? 409 : typeof frame.status === "number" ? frame.status : 200; - entry.resolve({ + const headers = frame.headers || {}; + if (frame.bodyByteCount === 0) { + // A zero-length body accepts no body_chunk, so the response completes now. + pending.delete(frame.id); + responseAssembly.delete(frame.id); + clearTimeout(entry.timer); + entry.resolve({ status: statusCode, headers: headers, body: "" }); + return; + } + // The body rides body_chunk frames that share this id. Record the envelope + // and wait for the chunks. + responseAssembly.set(frame.id, { status: statusCode, - headers: frame.headers || {}, - body: typeof frame.body === "string" ? frame.body : "", + headers: headers, + bodyByteCount: frame.bodyByteCount, + received: 0, + nextSeq: 0, + chunks: [], }); return; } + if (frame.type === "body_chunk") { + const asm = responseAssembly.get(frame.id); + if (!asm) return; + if (frame.seq !== asm.nextSeq) { + failRequest(frame.id, "duplex response body_chunk seq is out of order"); + return; + } + const decoded = Buffer.from(frame.data, "base64"); + if (decoded.toString("base64") !== frame.data) { + failRequest(frame.id, "duplex response body_chunk is not canonical base64"); + return; + } + if (decoded.length === 0) { + failRequest(frame.id, "duplex response body_chunk is empty"); + return; + } + if (asm.received + decoded.length > asm.bodyByteCount) { + failRequest(frame.id, "duplex response body overruns the declared size"); + return; + } + const isFinal = asm.received + decoded.length === asm.bodyByteCount; + if ( + decoded.length > DUPLEX_BODY_CHUNK_RAW_BYTES || + (!isFinal && decoded.length !== DUPLEX_BODY_CHUNK_RAW_BYTES) + ) { + failRequest(frame.id, "duplex response body_chunk has the wrong size"); + return; + } + asm.nextSeq += 1; + asm.received += decoded.length; + asm.chunks.push(decoded); + if (asm.received === asm.bodyByteCount) { + const entry = pending.get(frame.id); + responseAssembly.delete(frame.id); + if (!entry) return; + pending.delete(frame.id); + clearTimeout(entry.timer); + entry.resolve({ + status: asm.status, + headers: asm.headers, + body: Buffer.concat(asm.chunks).toString("utf8"), + }); + } + return; + } if (frame.type === "close") { triggerLoss("host sent a close frame"); return; @@ -2210,7 +2394,7 @@ function runDuplexGateway() { // local action here. } - const decoder = new DuplexFrameDecoder(); + const decoder = new DuplexFrameDecoder({ maxAggregateBytes: maxDuplexDecoderBytes }); process.stdin.on("data", (chunk) => { lastInboundAt = Date.now(); for (const result of decoder.push(chunk)) { @@ -2258,11 +2442,14 @@ function runDuplexGateway() { return; } const requestId = randomUUID(); - const requestBody = await readBody(req); + const requestBodyBuffer = Buffer.from(await readBody(req), "utf8"); if (unavailable) { writeJsonResponse(res, 503, { error: "bridge_unavailable" }); return; } + // The request body rides body_chunk frames. The envelope carries only the + // raw byte count, so the envelope stays small and the body splits into + // fixed-size slices that each stay under the frame bound. const requestFrame = { version: DUPLEX_FRAME_VERSION, type: "request", @@ -2271,10 +2458,36 @@ function runDuplexGateway() { path: url.pathname, query: url.search, headers: normalizeHeaders(req.headers), - body: requestBody, + bodyByteCount: requestBodyBuffer.length, }; + const encodedRequest = encodeDuplexFrameChecked(requestFrame); + if (!encodedRequest.ok) { + writeJsonResponse(res, 413, { error: "request_too_large" }); + return; + } + // Pre-encode every body_chunk frame and enforce the frame size bound on + // each. A fixed raw slice never exceeds the bound, so this guard is + // defensive. On a rejection, fail this one local request with a clean 413. + // The frames never leave the gateway, so no other in-flight request is + // affected and the channel stays open. + const chunkFrames = splitDuplexBodyIntoChunks(requestId, requestBodyBuffer); + const encodedChunks = []; + let chunkTooLarge = false; + for (const chunk of chunkFrames) { + const encodedChunk = encodeDuplexFrameChecked(chunk); + if (!encodedChunk.ok) { + chunkTooLarge = true; + break; + } + encodedChunks.push(encodedChunk.line); + } + if (chunkTooLarge) { + writeJsonResponse(res, 413, { error: "request_too_large" }); + return; + } const response = await new Promise((resolve) => { const timer = setTimeout(() => { + responseAssembly.delete(requestId); if (pending.delete(requestId)) { resolve({ status: 502, @@ -2284,7 +2497,8 @@ function runDuplexGateway() { } }, responseTimeoutMs); pending.set(requestId, { resolve: resolve, timer: timer }); - writeFrame(requestFrame); + process.stdout.write(encodedRequest.line); + for (const line of encodedChunks) process.stdout.write(line); }); res.statusCode = typeof response.status === "number" ? response.status : 200; for (const [key, value] of Object.entries(response.headers || {})) { diff --git a/packages/adapters/codex-local/src/server/device-login-parse.test.ts b/packages/adapters/codex-local/src/server/device-login-parse.test.ts index cfd6fa9b981..8768d183321 100644 --- a/packages/adapters/codex-local/src/server/device-login-parse.test.ts +++ b/packages/adapters/codex-local/src/server/device-login-parse.test.ts @@ -42,6 +42,62 @@ describe("parseDeviceLoginPrompt", () => { expect(result?.code).toBe("ABCD-EFGHJ"); }); + it("parse_returns_url_and_code_from_carriage_return_line_endings", () => { + // A pseudo-terminal ends each line with a carriage return and a line feed. + // The carriage return is whitespace, so the URL token stops before it, and the + // code line trims it. The parser reads the tokens the same as line-feed output. + const text = [ + "1. Open this link in your browser and sign in to your account", + EXACT_URL, + "2. Enter this one-time code (expires in 15 minutes)", + "ABCD-EFGHJ", + ].join("\r\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_returns_url_and_code_with_a_terminal_title_osc_sequence", () => { + // A pseudo-terminal can emit an Operating System Command (OSC) title sequence + // in the stream. The device-auth prompt still prints the URL and the code as + // plain standalone tokens on their own lines. The OSC title sequence carries no + // URL token, so it does not shift the URL or the code, and the parser reads + // both tokens. + const osc = "\x1b]0;Codex\x07"; + const text = [ + `${osc}1. Open this link in your browser and sign in to your account`, + EXACT_URL, + "2. Enter this one-time code (expires in 15 minutes)", + "ABCD-EFGHJ", + ].join("\n"); + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + + it("parse_returns_url_and_code_from_a_merged_raw_pseudo_terminal_frame", () => { + // One buffer holds the whole prompt as a real pseudo-terminal emits it: a + // leading title Operating System Command (OSC) sequence, color Control + // Sequence Introducer (CSI) sequences around the URL and the code, and + // carriage-return line endings. The parser removes the color sequences and + // reads the plain tokens. + const title = "\x1b]0;Codex login\x07"; + const cyan = "\x1b[36m"; + const bold = "\x1b[1m"; + const reset = "\x1b[0m"; + const text = + `${title}1. Open this link in your browser and sign in to your account\r\n` + + `${cyan}${EXACT_URL}${reset}\r\n` + + `2. Enter this one-time code (expires in 15 minutes)\r\n` + + `${bold}ABCD-EFGHJ${reset}\r\n`; + const result = parseDeviceLoginPrompt(text); + expect(result).not.toBeNull(); + expect(result?.url).toBe(EXACT_URL); + expect(result?.code).toBe("ABCD-EFGHJ"); + }); + it("parse_returns_null_when_prompt_absent", () => { const text = "Some unrelated log line\nNothing to see here\n"; expect(parseDeviceLoginPrompt(text)).toBeNull(); diff --git a/packages/adapters/codex-local/src/server/device-login-runner.test.ts b/packages/adapters/codex-local/src/server/device-login-runner.test.ts index 7a149581499..b9393e8bbad 100644 --- a/packages/adapters/codex-local/src/server/device-login-runner.test.ts +++ b/packages/adapters/codex-local/src/server/device-login-runner.test.ts @@ -24,10 +24,10 @@ interface FakeDriverOptions { function createFakeDriver(options: FakeDriverOptions = {}) { const disposeCalls = { count: 0 }; const driver: SandboxLoginDriver = { - async execStreaming(_command, onStdout) { + async start(_command, onData) { if (options.execError) throw options.execError; for (const chunk of options.chunks ?? []) { - onStdout(chunk); + onData(chunk); } if (options.hang) { // Never resolve. The runner must fall back to its timeout or signal. diff --git a/packages/adapters/codex-local/src/server/device-login-runner.ts b/packages/adapters/codex-local/src/server/device-login-runner.ts index 4c66a702c0e..968031e1130 100644 --- a/packages/adapters/codex-local/src/server/device-login-runner.ts +++ b/packages/adapters/codex-local/src/server/device-login-runner.ts @@ -38,18 +38,33 @@ const MAX_PARSE_BUFFER_CHARS = 64 * 1024; /** * The sandbox side of the device-login run. The runner never calls Daytona - * directly; a caller injects a concrete driver. A production driver binds these - * three methods to a non-persisting Daytona exec path, a file read, and a - * sandbox delete. + * directly; a caller injects a concrete driver. A production driver binds `start` + * to the shared login pseudo-terminal (PTY) transport, binds `readFile` to a + * descriptor-bound credential read, and binds `dispose` to the transport dispose. + * + * The `start` shape matches the shared login pseudo-terminal transport start + * method, so the driver runs the login command on a real pseudo-terminal. The + * login command + * needs a pseudo-terminal: pipe stdio emits no login prompt. The runner never + * calls `stop` or `write` on the driver; the device-login flow needs no delayed + * input, and the service owns the sandbox delete. The runner disposes the driver + * one time on every terminal state. */ export interface SandboxLoginDriver extends LoginRunnerDisposable { /** - * Runs `command` in the sandbox and streams standard output to `onStdout` in - * memory. Resolves with the command exit code when the command ends. A driver - * must not persist the raw output to any durable log. + * Starts `command` on a pseudo-terminal and streams the terminal output to + * `onData` in memory, in order, as the pseudo-terminal emits it. Resolves with + * the command exit code when the command ends. A driver must not persist the raw + * output to any durable log. + */ + start(command: string, onData: (chunk: string) => void): Promise<{ exitCode: number | null }>; + /** + * Reads the credential bytes with one descriptor-bound read. The read is + * separate from the pseudo-terminal session; the session has no file-read + * method. A driver runs one fixed, server-controlled operation that opens the + * verified session home and the credential file with no symlink follow, checks + * the opened descriptor, and reads only from that same descriptor. */ - execStreaming(command: string, onStdout: (chunk: string) => void): Promise<{ exitCode: number | null }>; - /** Reads the bytes of one file from the sandbox. */ readFile(path: string): Promise; } @@ -123,8 +138,8 @@ export async function runDeviceLogin( return { outcome: "cancelled", exitCode: null, promptSurfaced }; } - const exec = driver.execStreaming(command, onStdout); - const raced = await raceLoginRunnerExit(exec, timeoutMs, signal); + const started = driver.start(command, onStdout); + const raced = await raceLoginRunnerExit(started, timeoutMs, signal); if (raced.kind === "timeout") { log("[paperclip] Device login timed out; disposing the sandbox."); diff --git a/packages/plugins/sandbox-providers/daytona/package.json b/packages/plugins/sandbox-providers/daytona/package.json index ddc5acd934f..666300e93cc 100644 --- a/packages/plugins/sandbox-providers/daytona/package.json +++ b/packages/plugins/sandbox-providers/daytona/package.json @@ -42,7 +42,7 @@ ], "scripts": { "prebuild": "pnpm -C ../../../.. --filter @paperclipai/plugin-sdk ensure-build-deps", - "build": "rm -rf dist && tsc", + "build": "rm -rf dist && tsc && mkdir -p dist/scripts && cp -R src/scripts/. dist/scripts/", "clean": "rm -rf dist", "typecheck": "pnpm -C ../../../.. --filter @paperclipai/plugin-sdk ensure-build-deps && tsc --noEmit", "test": "vitest run --config vitest.config.ts", diff --git a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts index 1610a3d83e2..19339783f52 100644 --- a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.test.ts @@ -7,6 +7,7 @@ import { type DaytonaPtyHandle, type DaytonaPtyProcess, } from "./duplex-command-stream.js"; +import { PTY_INPUT_CHUNK_BYTES, PTY_MESSAGE_CAP_BYTES } from "./pty-chunked-input.js"; // The carriage return that submits the launch wrapper line to the terminal. const ENTER = "\r"; @@ -273,6 +274,8 @@ describe("openDaytonaDuplexChannelSession", () => { // The launch wrapper is the first input; the host frame write is the second. session.write('{"version":1,"type":"heartbeat"}\n'); + // The chunker awaits each send, so let the write settle before the assertion. + await new Promise((resolve) => setImmediate(resolve)); expect(process.handle?.inputs[1]).toBe('{"version":1,"type":"heartbeat"}\n'); // The fake terminal echoes nothing, so the write produces no data callback. // A real terminal in raw mode with echo off behaves the same way. @@ -357,22 +360,104 @@ describe("openDaytonaDuplexChannelSession", () => { expect(received).toEqual(['{"version":1,"type":"heartbeat"}\n']); }); - it("resolves wait with the gateway exit code", async () => { + it("maps a numeric SDK exit code to a process exit", async () => { const process = createFakeProcess(); const session = await openDaytonaDuplexChannelSession(process, GATEWAY); process.handle?.finish(3); - await expect(session.wait()).resolves.toEqual({ exitCode: 3 }); + // A numeric exit code is a real process exit, not a transport close. + await expect(session.wait()).resolves.toEqual({ exitCode: 3, transportClosed: false }); }); - it("maps an absent exit code to null", async () => { + it("maps an absent SDK exit code to a transport close", async () => { const process = createFakeProcess(); const session = await openDaytonaDuplexChannelSession(process, GATEWAY); process.handle?.finish(undefined); - await expect(session.wait()).resolves.toEqual({ exitCode: null }); + // A non-numeric SDK result marks a reason-less transport close with no exit + // data, so the seam reports a transport close, not a process exit. + await expect(session.wait()).resolves.toEqual({ exitCode: null, transportClosed: true }); + }); + + it("splits a write above the provider cap into more than one send under the cap", async () => { + const process = createFakeProcess(); + const session = await openDaytonaDuplexChannelSession(process, GATEWAY); + const handle = process.handle; + if (!handle) throw new Error("The fake process opened no handle."); + // The launch wrapper is the first recorded input. Count only the write's sends. + const before = handle.inputs.length; + + // A fixed payload well above the provider message cap. It stays fixed, so a + // raised chunk size above this size makes the write one send and fails this + // test. The payload is ASCII, so the fake's per-send decode keeps each byte. + const payload = "x".repeat(150_416); + session.write(payload); + // The chunker awaits each send, so let the write settle before the assertion. + await new Promise((resolve) => setImmediate(resolve)); + + const sends = handle.inputs.slice(before); + expect(sends.length).toBeGreaterThan(1); + for (const chunk of sends) { + expect(Buffer.byteLength(chunk, "utf8")).toBeLessThanOrEqual(PTY_INPUT_CHUNK_BYTES); + expect(Buffer.byteLength(chunk, "utf8")).toBeLessThanOrEqual(PTY_MESSAGE_CAP_BYTES); + } + // The sends rejoin to the exact payload, so the chunker loses no byte. + expect(sends.join("")).toBe(payload); + }); + + it("sends a write below the provider cap as exactly one send", async () => { + const process = createFakeProcess(); + const session = await openDaytonaDuplexChannelSession(process, GATEWAY); + const handle = process.handle; + if (!handle) throw new Error("The fake process opened no handle."); + const before = handle.inputs.length; + + session.write('{"version":1,"type":"heartbeat"}\n'); + // The chunker awaits each send, so let the write settle before the assertion. + await new Promise((resolve) => setImmediate(resolve)); + + expect(handle.inputs.slice(before)).toHaveLength(1); + }); + + it("keeps two writes in order and never interleaves their chunks", async () => { + // A handle whose `sendInput` yields a microtask before it records the chunk. + // The yield opens a window where a second write could jump ahead. The session + // chains the writes, so the second write starts only after the first ends. + const inputs: string[] = []; + const handle: DaytonaPtyHandle = { + async waitForConnection(): Promise {}, + async sendInput(data: string | Uint8Array): Promise { + await Promise.resolve(); + inputs.push(typeof data === "string" ? data : new TextDecoder().decode(data)); + }, + wait(): Promise<{ exitCode?: number; error?: string }> { + return new Promise(() => {}); + }, + async kill(): Promise {}, + async disconnect(): Promise {}, + }; + const process: DaytonaPtyProcess = { + async createPty(): Promise { + return handle; + }, + }; + const session = await openDaytonaDuplexChannelSession(process, GATEWAY); + // Two payloads above the cap, so each write is more than one chunk. "a" marks + // the first write and "b" marks the second write. + const first = "a".repeat(150_416); + const second = "b".repeat(150_416); + + session.write(first); + session.write(second); + // Let both writes settle. + await new Promise((resolve) => setImmediate(resolve)); + + // Drop the launch wrapper (the first input) and rejoin the chunks. The rejoin + // equals the first payload then the second payload, so no chunk of the second + // write lands between the chunks of the first write. + expect(inputs.slice(1).join("")).toBe(`${first}${second}`); }); it("kills the child on stop and releases the socket on close", async () => { @@ -475,6 +560,55 @@ describe("openDaytonaDuplexChannelSession", () => { expect(writeErrors).toEqual(["write_error"]); expect(killed).toBe(1); }); + + it("does not send a queued write after a first-send rejection terminalizes the channel", async () => { + // Two writes queue on the chain. The launch wrapper write succeeds, then the + // first frame write rejects and terminalizes the channel. The second queued + // write must not reach `sendInput`, so no write runs on the closed transport. + const sends: string[] = []; + let killed = 0; + const handle: DaytonaPtyHandle = { + async waitForConnection(): Promise {}, + async sendInput(data: string | Uint8Array): Promise { + const text = typeof data === "string" ? data : new TextDecoder().decode(data); + sends.push(text); + // The launch wrapper write (the first send) succeeds. The first frame + // write (the second send) rejects and terminalizes the channel. + if (sends.length === 1) return; + throw new Error("broken"); + }, + wait(): Promise<{ exitCode?: number; error?: string }> { + return new Promise(() => {}); + }, + async kill(): Promise { + killed += 1; + }, + async disconnect(): Promise {}, + }; + const process: DaytonaPtyProcess = { + async createPty(): Promise { + return handle; + }, + }; + const writeErrors: string[] = []; + const session = await openDaytonaDuplexChannelSession(process, GATEWAY, { + onWriteError: (reason) => writeErrors.push(reason), + }); + + session.write("first\n"); + session.write("second\n"); + // Let the write chain settle both queued writes. + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + + // The channel terminalized one time on the first frame write. The second + // queued write sent no chunk, so exactly two sends ran: the launch wrapper and + // the first frame write. The second frame never reached the transport. + expect(writeErrors).toEqual(["write_error"]); + expect(killed).toBe(1); + expect(sends).toHaveLength(2); + expect(sends.some((text) => text.includes("second"))).toBe(false); + }); }); describe("createDaytonaDuplexChannelSessionOpener", () => { diff --git a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts index 2aba7b76c8c..3da94be8fc1 100644 --- a/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts +++ b/packages/plugins/sandbox-providers/daytona/src/duplex-command-stream.ts @@ -27,7 +27,7 @@ // Dependency boundary: this provider plugin ships standalone (the workspace // excludes `packages/plugins/sandbox-providers/**`). So the module imports no // workspace package. It reuses the narrow Daytona PTY surface that -// `setup-token-pty.ts` declares (`DaytonaPtyProcess`, `DaytonaPtyHandle`, +// `login-pty.ts` declares (`DaytonaPtyProcess`, `DaytonaPtyHandle`, // `DaytonaPtyCreateOptions`), so a caller passes `sandbox.process` with no // adapter. The narrow surface keeps the module unit-testable with a fake PTY. @@ -37,13 +37,15 @@ import type { DaytonaPtyCreateOptions, DaytonaPtyHandle, DaytonaPtyProcess, -} from "./setup-token-pty.js"; +} from "./login-pty.js"; export type { DaytonaPtyCreateOptions, DaytonaPtyHandle, DaytonaPtyProcess, -} from "./setup-token-pty.js"; +} from "./login-pty.js"; + +import { sendPtyInputInChunks } from "./pty-chunked-input.js"; /** * A live duplex channel session for one command. The session allocates a real @@ -56,8 +58,13 @@ export interface DuplexChannelSession { onData(listener: (chunk: string) => void): void; /** Writes raw input bytes to the pseudo-terminal. */ write(data: string): void; - /** Resolves with the child exit code when the command ends. */ - wait(): Promise<{ exitCode: number | null }>; + /** + * Resolves when the command ends or the transport closes. A numeric `exitCode` + * is a real process exit. `transportClosed` is true when the pseudo-terminal + * socket closed with no exit data, so the caller can tell a real process exit + * from a reason-less transport close. + */ + wait(): Promise<{ exitCode: number | null; transportClosed: boolean }>; /** Stops the child process. Safe to call more than one time. */ kill(): void; /** Releases the session resources. Safe to call more than one time. */ @@ -203,14 +210,28 @@ export async function openDaytonaDuplexChannelSession( // Report a host-to-sandbox write failure one time and end the channel at once. // The seam carries only the typed reason; the raw provider error never leaves // this scope. The channel end propagates the loss up through the exit. + // + // `channelTerminated` turns true on the first rejected chunk, before the early + // return, so a later queued write reads it and sends no chunk. Without the flag + // a queued write runs after the terminal kill and calls `sendInput` on the + // closed transport. + let channelTerminated = false; let writeErrorReported = false; const endOnWriteError = (): void => { + channelTerminated = true; if (writeErrorReported) return; writeErrorReported = true; options?.onWriteError?.(DAYTONA_DUPLEX_WRITE_ERROR_REASON); void handle.kill().catch(() => undefined); }; + // The tail of the write chain. The chunker awaits each chunk, so one write can + // suspend between its chunks. The chain runs each write after the previous + // write ends, so the chunks of two writes never interleave on the wire. The + // chain never rejects, because the chunker maps a rejected send to + // `endOnWriteError` and returns. + let writeChain: Promise = Promise.resolve(); + return { onData(next: (chunk: string) => void): void { listener = next; @@ -221,17 +242,38 @@ export async function openDaytonaDuplexChannelSession( } }, write(data: string): void { - // Fire the input write. A write error must not throw into the transport, so - // the transport's stream stays the single result path. On a rejected write, - // end the channel at once and report the typed reason. The raw provider - // error never reaches a sink. - void handle.sendInput(data).catch(() => { - endOnWriteError(); + // Send the input as byte-bounded chunks under the provider message cap. A + // whole payload in one message can cross the cap and take the channel down, + // so the chunker slices the payload and sends each chunk in order. The chain + // runs this write after the previous write ends, so two writes keep their + // order and never interleave their chunks. A write error must not throw into + // the transport, so the transport's stream stays the single result path. On + // a rejected chunk, the chunker ends the channel one time through + // `endOnWriteError` and sends no later chunk. The raw provider error never + // reaches a sink. + // + // A rejected chunk terminalizes the channel and kills the transport. So this + // write skips the send when `channelTerminated` is true, and a queued write + // never calls `sendInput` on the closed transport. + writeChain = writeChain.then(() => { + if (channelTerminated) return; + return sendPtyInputInChunks( + (chunk) => handle.sendInput(chunk), + data, + endOnWriteError, + ); }); }, - async wait(): Promise<{ exitCode: number | null }> { + async wait(): Promise<{ exitCode: number | null; transportClosed: boolean }> { const result = await handle.wait(); - return { exitCode: typeof result.exitCode === "number" ? result.exitCode : null }; + if (typeof result.exitCode === "number") { + // A numeric exit code is a real process exit. The SDK parses it from the + // pseudo-terminal WebSocket close reason (for example `{"exitCode":0}`). + return { exitCode: result.exitCode, transportClosed: false }; + } + // A non-numeric exit code marks a reason-less transport close: the socket + // closed with no exit data. It is a transport close, not a process exit. + return { exitCode: null, transportClosed: true }; }, kill(): void { void handle.kill().catch(() => undefined); diff --git a/packages/plugins/sandbox-providers/daytona/src/login-pty.test.ts b/packages/plugins/sandbox-providers/daytona/src/login-pty.test.ts new file mode 100644 index 00000000000..398106dae33 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/login-pty.test.ts @@ -0,0 +1,645 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + composeLaunchLine, + createDaytonaLoginHomeFs, + createDaytonaLoginPtySessionOpener, + encodePosixShellArg, + openDaytonaLoginPtySession, + type DaytonaExecResult, + type DaytonaLoginHomeFs, + type DaytonaPtyCreateOptions, + type DaytonaPtyHandle, + type DaytonaPtyProcess, + type DaytonaSandboxExec, + type LoginHomeInspection, + type LoginPtyLaunchDescriptor, +} from "./login-pty.js"; + +// The Enter byte the terminal login UI reads to submit the browser code. +const ENTER = "\r"; + +const HOME = "/tmp/paperclip-adapter-login/11111111-2222-4333-8444-555555555555"; +const LOGIN_UID = 1000; + +const CLAUDE: LoginPtyLaunchDescriptor = { loginCommandKey: "claude", sessionHome: HOME }; +const CODEX: LoginPtyLaunchDescriptor = { loginCommandKey: "codex", sessionHome: HOME }; + +/** A directory entry in the virtual filesystem. */ +type FakeEntry = Omit; + +const ABSENT: LoginHomeInspection = { + exists: false, + isSymlink: false, + isDirectory: false, + mode: "", + ownerUid: null, +}; + +/** + * A fake login-home filesystem. It models one virtual path. `createDirectory` + * fails when the path exists and otherwise stores `createAs` (a clean 0700 + * directory owned by the login user by default). A test seeds a pre-existing + * entry, forces a bad created entry, or swaps the entry through `onCreatePty`, so + * a test drives each rejection and the pre-launch re-check. + */ +function createFakeHomeFs(config?: { + loginUid?: number; + seed?: FakeEntry; + createAs?: FakeEntry; + createShouldFail?: boolean; +}): DaytonaLoginHomeFs & { + created: Array<{ path: string; mode: string }>; + inspectCount: number; + setEntry: (entry: FakeEntry | null) => void; +} { + const loginUid = config?.loginUid ?? LOGIN_UID; + const cleanDir: FakeEntry = { + isSymlink: false, + isDirectory: true, + mode: "700", + ownerUid: loginUid, + }; + let entry: FakeEntry | null = config?.seed ?? null; + const created: Array<{ path: string; mode: string }> = []; + const state = { inspectCount: 0 }; + return { + created, + get inspectCount() { + return state.inspectCount; + }, + setEntry(next: FakeEntry | null): void { + entry = next; + }, + async loginUserId(): Promise { + return loginUid; + }, + async inspect(): Promise { + state.inspectCount += 1; + return entry ? { exists: true, ...entry } : ABSENT; + }, + async createDirectory(path: string, mode: string): Promise { + if (config?.createShouldFail || entry) { + throw new Error("LOGIN_PTY_HOME_REJECTED"); + } + created.push({ path, mode }); + entry = config?.createAs ?? cleanDir; + }, + }; +} + +/** + * A fake Daytona PTY handle. It records each input write, drives the output + * stream on demand, and records the kill and the disconnect. The tests use it in + * place of the real SDK `PtyHandle`, so the session runs with no sandbox. + */ +function createFakePtyHandle( + onData: (data: Uint8Array) => void | Promise, + onWaitForConnection?: () => void, +): DaytonaPtyHandle & { + inputs: string[]; + emitText: (text: string) => void; + emitBytes: (bytes: Uint8Array) => void; + finish: (exitCode: number | undefined) => void; + killed: number; + disconnected: number; +} { + const encoder = new TextEncoder(); + const inputs: string[] = []; + let resolveWait: ((value: { exitCode?: number; error?: string }) => void) | null = null; + const waitPromise = new Promise<{ exitCode?: number; error?: string }>((resolve) => { + resolveWait = resolve; + }); + let killed = 0; + let disconnected = 0; + return { + async waitForConnection(): Promise { + onWaitForConnection?.(); + }, + async sendInput(data: string | Uint8Array): Promise { + inputs.push(typeof data === "string" ? data : new TextDecoder().decode(data)); + }, + wait(): Promise<{ exitCode?: number; error?: string }> { + return waitPromise; + }, + async kill(): Promise { + killed += 1; + }, + async disconnect(): Promise { + disconnected += 1; + }, + emitText(text: string): void { + onData(encoder.encode(text)); + }, + emitBytes(bytes: Uint8Array): void { + onData(bytes); + }, + finish(exitCode: number | undefined): void { + resolveWait?.({ exitCode }); + }, + get inputs(): string[] { + return inputs; + }, + get killed(): number { + return killed; + }, + get disconnected(): number { + return disconnected; + }, + }; +} + +/** + * A fake Daytona process. It opens one fake PTY handle and records the create + * options, so a test asserts the terminal size and the launch line. `onCreatePty` + * runs when the session opens the terminal, so a test swaps a symlink in after + * the directory creation but before the launch input. + */ +function createFakeProcess(onCreatePty?: () => void): DaytonaPtyProcess & { + handle: ReturnType | null; + createOptions: DaytonaPtyCreateOptions | null; + createCount: number; +} { + const state: { + handle: ReturnType | null; + createOptions: DaytonaPtyCreateOptions | null; + createCount: number; + } = { handle: null, createOptions: null, createCount: 0 }; + return { + get handle() { + return state.handle; + }, + get createOptions() { + return state.createOptions; + }, + get createCount() { + return state.createCount; + }, + async createPty(options: DaytonaPtyCreateOptions): Promise { + state.createCount += 1; + state.createOptions = options; + onCreatePty?.(); + const handle = createFakePtyHandle(options.onData); + state.handle = handle; + return handle; + }, + }; +} + +describe("encodePosixShellArg", () => { + it("wraps a value in single quotes and neutralizes metacharacters", () => { + expect(encodePosixShellArg("/tmp/x")).toBe("'/tmp/x'"); + // A metacharacter, a space, and a semicolon stay literal inside the quotes. + expect(encodePosixShellArg("; rm -rf / #")).toBe("'; rm -rf / #'"); + // An embedded single quote closes, escapes, and reopens the quote. + expect(encodePosixShellArg("a'b")).toBe("'a'\\''b'"); + expect(encodePosixShellArg("$(id)")).toBe("'$(id)'"); + }); +}); + +describe("composeLaunchLine", () => { + it("composes the Claude line with no CODEX_HOME", () => { + const line = composeLaunchLine(CLAUDE); + expect(line).toBe("exec claude setup-token"); + expect(line).not.toContain("CODEX_HOME"); + }); + + it("composes the Codex line with exactly one encoded CODEX_HOME", () => { + const line = composeLaunchLine(CODEX); + expect(line).toBe(`exec env CODEX_HOME='${HOME}' codex login --device-auth`); + // Exactly one CODEX_HOME assignment, and the exact approved Codex command. + expect(line.match(/CODEX_HOME=/g)).toHaveLength(1); + expect(line.endsWith("codex login --device-auth")).toBe(true); + }); +}); + +describe("openDaytonaLoginPtySession — session home", () => { + it("creates the exact UUID directory as a new 0700 directory owned by the login user", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await openDaytonaLoginPtySession(process, fs, CLAUDE); + + // The provider created the exact directory as a new 0700 directory. + expect(fs.created).toEqual([{ path: HOME, mode: "700" }]); + // The terminal opened and the launch line ran. + expect(process.createCount).toBe(1); + expect(process.handle?.inputs[0]).toBe("exec claude setup-token" + ENTER); + }); + + it("sends the Codex launch line with the encoded CODEX_HOME", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await openDaytonaLoginPtySession(process, fs, CODEX); + + expect(process.handle?.inputs[0]).toBe( + `exec env CODEX_HOME='${HOME}' codex login --device-auth` + ENTER, + ); + // The Claude line carries no CODEX_HOME; the Codex line carries exactly one. + expect((process.handle?.inputs[0]?.match(/CODEX_HOME=/g) ?? []).length).toBe(1); + }); + + it("rejects a pre-existing target before it creates the directory", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + seed: { isSymlink: false, isDirectory: true, mode: "700", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + // The provider never created the directory and never opened the terminal. + expect(fs.created).toEqual([]); + expect(process.createCount).toBe(0); + }); + + it("rejects a pre-existing symlink at the target", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + seed: { isSymlink: true, isDirectory: false, mode: "777", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created target that is a symlink", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: true, isDirectory: false, mode: "700", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created target that is not a directory", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: false, isDirectory: false, mode: "700", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created directory with a wrong owner", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: false, isDirectory: true, mode: "700", ownerUid: LOGIN_UID + 1 }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a created directory with a wrong mode", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs({ + createAs: { isSymlink: false, isDirectory: true, mode: "755", ownerUid: LOGIN_UID }, + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + expect(process.createCount).toBe(0); + }); + + it("rejects a symlink swapped in after creation, before the launch input", async () => { + // The directory validates cleanly, then a symlink replaces it while the + // terminal opens. The no-symlink re-check before the launch input fails. + const fs = createFakeHomeFs(); + const process = createFakeProcess(() => { + fs.setEntry({ isSymlink: true, isDirectory: false, mode: "777", ownerUid: LOGIN_UID }); + }); + + await expect(openDaytonaLoginPtySession(process, fs, CLAUDE)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + // The terminal opened, but the launch input never ran. + expect(process.createCount).toBe(1); + expect(process.handle?.inputs).toEqual([]); + }); + + it("rejects a descriptor with a command key outside the closed set", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await expect( + openDaytonaLoginPtySession(process, fs, { + loginCommandKey: "gemini" as unknown as "claude", + sessionHome: HOME, + }), + ).rejects.toThrow("LOGIN_PTY_DESCRIPTOR_REJECTED"); + // The provider never touched the filesystem or the terminal. + expect(fs.created).toEqual([]); + expect(process.createCount).toBe(0); + }); + + it("rejects a descriptor whose session home shape is wrong", async () => { + const process = createFakeProcess(); + const fs = createFakeHomeFs(); + + await expect( + openDaytonaLoginPtySession(process, fs, { + loginCommandKey: "codex", + sessionHome: "/tmp/paperclip-adapter-login/../etc", + }), + ).rejects.toThrow("LOGIN_PTY_DESCRIPTOR_REJECTED"); + expect(process.createCount).toBe(0); + }); +}); + +describe("openDaytonaLoginPtySession — session mechanics", () => { + it("starts the login command on a pseudo-terminal with a fixed size", async () => { + const process = createFakeProcess(); + + await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + + expect(process.createOptions?.cols).toBe(120); + expect(process.createOptions?.rows).toBe(30); + expect(process.handle?.inputs[0]).toBe("exec claude setup-token" + ENTER); + }); + + it("returns incremental terminal output to the listener", async () => { + const process = createFakeProcess(); + const received: string[] = []; + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.onData((chunk) => received.push(chunk)); + + process.handle?.emitText("the url below to sign in\n"); + process.handle?.emitText("Paste code here if prompted\n"); + + expect(received).toEqual(["the url below to sign in\n", "Paste code here if prompted\n"]); + }); + + it("buffers early output until the listener registers", async () => { + const process = createFakeProcess(); + const received: string[] = []; + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + process.handle?.emitText("early output "); + process.handle?.emitText("more output"); + expect(received).toEqual([]); + + session.onData((chunk) => received.push(chunk)); + expect(received).toEqual(["early output more output"]); + }); + + it("delivers the browser code plus the Enter byte to the command", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.write("BROWSERCODE" + ENTER); + // The write runs through the serialized chunk chain, so it lands on a later + // microtask. Flush the chain before the assertion. + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(process.handle?.inputs[1]).toBe("BROWSERCODE" + ENTER); + expect(process.handle?.inputs[1]?.endsWith(ENTER)).toBe(true); + }); + + it("keeps a multibyte character whole across two output chunks", async () => { + const process = createFakeProcess(); + const received: string[] = []; + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.onData((chunk) => received.push(chunk)); + + const euro = new TextEncoder().encode("€"); + process.handle?.emitBytes(euro.subarray(0, 2)); + process.handle?.emitBytes(euro.subarray(2)); + + expect(received.join("")).toBe("€"); + }); + + it("resolves wait with the command exit code", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + process.handle?.finish(9); + + await expect(session.wait()).resolves.toEqual({ exitCode: 9 }); + }); + + it("maps an absent exit code to null", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + process.handle?.finish(undefined); + + await expect(session.wait()).resolves.toEqual({ exitCode: null }); + }); + + it("kills the child and closes the session", async () => { + const process = createFakeProcess(); + + const session = await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE); + session.kill(); + expect(process.handle?.killed).toBe(1); + + await session.close(); + expect(process.handle?.disconnected).toBe(1); + }); + + it("passes the working directory to the pseudo-terminal", async () => { + const process = createFakeProcess(); + + await openDaytonaLoginPtySession(process, createFakeHomeFs(), CLAUDE, { cwd: "/workspace" }); + + expect(process.createOptions?.cwd).toBe("/workspace"); + }); +}); + +describe("openDaytonaLoginPtySession — ancestor symlink", () => { + it("starts no pseudo-terminal when the home filesystem check fails closed", async () => { + // A symlinked login-root ancestor makes the descriptor-relative inspection fail + // closed. The session opener must reject before it opens the terminal, so no + // pseudo-terminal starts and the login command never runs. + const process = createFakeProcess(); + let created = false; + const fs: DaytonaLoginHomeFs = { + async loginUserId(): Promise { + return LOGIN_UID; + }, + async inspect(): Promise { + throw new Error("LOGIN_PTY_HOME_REJECTED"); + }, + async createDirectory(): Promise { + created = true; + }, + }; + + await expect(openDaytonaLoginPtySession(process, fs, CODEX)).rejects.toThrow( + "LOGIN_PTY_HOME_REJECTED", + ); + // The provider never created the directory and never opened the terminal. + expect(created).toBe(false); + expect(process.createCount).toBe(0); + }); +}); + +describe("createDaytonaLoginHomeFs — login-profile preamble", () => { + // A capturing exec surface. It records each command and returns a fixed result, + // so a test reads the exact command string the helper runs. + function createCapturingExec(result: DaytonaExecResult): DaytonaSandboxExec & { + commands: string[]; + } { + const commands: string[] = []; + return { + commands, + async executeCommand(command: string): Promise { + commands.push(command); + // `id -u` resolves the login user id. Return a fixed uid so the caller + // continues to the node helper command under test. + if (command.startsWith("id -u")) { + return { exitCode: 0, result: "1000" }; + } + return result; + }, + }; + } + + // The command sources /etc/profile before it runs node. The Daytona image may + // expose node only through a login profile, so node must run after the profile + // source. This assertion proves the helper runtime and the login PTY capability + // stay consistent. + function expectProfileBeforeNode(command: string | undefined): void { + expect(command).toBeDefined(); + const profileIndex = command!.indexOf("/etc/profile"); + const nodeIndex = command!.indexOf("node -e"); + expect(profileIndex).toBeGreaterThanOrEqual(0); + expect(nodeIndex).toBeGreaterThanOrEqual(0); + expect(profileIndex).toBeLessThan(nodeIndex); + } + + it("sources the login profiles before it runs the node inspect helper", async () => { + const exec = createCapturingExec({ exitCode: 0, result: "ABSENT" }); + const fs = createDaytonaLoginHomeFs(exec); + await fs.inspect(HOME); + expectProfileBeforeNode(exec.commands.find((command) => command.includes("node -e"))); + }); + + it("sources the login profiles before it runs the node create helper", async () => { + const exec = createCapturingExec({ exitCode: 0, result: "" }); + const fs = createDaytonaLoginHomeFs(exec); + await fs.createDirectory(HOME, "700"); + expectProfileBeforeNode(exec.commands.find((command) => command.includes("node -e"))); + }); +}); + +// The real login-home filesystem runs a node helper on the sandbox. The helper +// walks the path with `/proc/self/fd`, which is Linux only. A non-Linux host skips +// these tests. The fake exec runs the helper locally, so the descriptor-relative +// walk is tested against a real filesystem. The login sandbox is Linux. +const describeLinux = process.platform === "linux" ? describe : describe.skip; + +describeLinux("createDaytonaLoginHomeFs — descriptor-relative walk", () => { + let root: string; + + beforeAll(() => { + root = mkdtempSync(path.join(tmpdir(), "daytona-login-home-")); + }); + afterAll(() => { + rmSync(root, { recursive: true, force: true }); + }); + + // A local exec surface. It runs the composed command with `/bin/sh -c`, so the + // real node helper runs against the local filesystem. + function createLocalExec(): DaytonaSandboxExec { + return { + async executeCommand(command: string): Promise { + const result = spawnSync("/bin/sh", ["-c", command], { encoding: "utf8" }); + return { exitCode: result.status ?? undefined, result: result.stdout ?? "" }; + }, + }; + } + + const UUID = "11111111-2222-4333-8444-555555555555"; + + it("reports a not-yet-created target as absent", async () => { + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const home = path.join(root, "absent-root", UUID); + const inspection = await fs.inspect(home); + expect(inspection.exists).toBe(false); + }); + + it("creates the login root and the session home relative to a trusted descriptor", async () => { + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const home = path.join(root, "clean-root", UUID); + await fs.createDirectory(home, "700"); + const inspection = await fs.inspect(home); + expect(inspection.exists).toBe(true); + expect(inspection.isSymlink).toBe(false); + expect(inspection.isDirectory).toBe(true); + expect(inspection.mode).toBe("700"); + expect(inspection.ownerUid).toBe(process.getuid ? process.getuid() : inspection.ownerUid); + }); + + it("rejects a pre-existing session home target", async () => { + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const home = path.join(root, "exists-root", UUID); + await fs.createDirectory(home, "700"); + // A second create of the same target fails, because the target exists. + await expect(fs.createDirectory(home, "700")).rejects.toThrow("LOGIN_PTY_HOME_REJECTED"); + }); + + it("fails the inspection closed when the login root is a symlink", async () => { + // Pre-place a symlink at the login-root ancestor. The descriptor-relative walk + // opens the root with a no-follow open, so the inspection fails closed. + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const attacker = path.join(root, "inspect-attacker"); + mkdirSync(attacker, { mode: 0o700 }); + const rootLink = path.join(root, "inspect-symlink-root"); + symlinkSync(attacker, rootLink); + const home = path.join(rootLink, UUID); + await expect(fs.inspect(home)).rejects.toThrow("LOGIN_PTY_HOME_REJECTED"); + }); + + it("fails the creation closed when the login root is a pre-placed symlink", async () => { + // Pre-place a symlink at the login-root ancestor that points at an attacker + // tree. The creation opens the root with a no-follow open after the tolerant + // `mkdir`, so it fails closed and creates no directory in the attacker tree. + const fs = createDaytonaLoginHomeFs(createLocalExec()); + const attacker = path.join(root, "create-attacker"); + mkdirSync(attacker, { mode: 0o700 }); + const rootLink = path.join(root, "create-symlink-root"); + symlinkSync(attacker, rootLink); + const home = path.join(rootLink, UUID); + await expect(fs.createDirectory(home, "700")).rejects.toThrow("LOGIN_PTY_HOME_REJECTED"); + // The creation placed no session home inside the attacker tree. + expect(existsSync(path.join(attacker, UUID))).toBe(false); + }); +}); + +describe("createDaytonaLoginPtySessionOpener", () => { + it("opens a session for the descriptor on each call", async () => { + const process = createFakeProcess(); + const opener = createDaytonaLoginPtySessionOpener(process, createFakeHomeFs(), { + cwd: "/workspace", + }); + + const session = await opener(CLAUDE); + + expect(process.createOptions?.cwd).toBe("/workspace"); + expect(process.handle?.inputs[0]).toBe("exec claude setup-token" + ENTER); + expect(typeof session.onData).toBe("function"); + expect(typeof session.write).toBe("function"); + expect(typeof session.wait).toBe("function"); + expect(typeof session.kill).toBe("function"); + expect(typeof session.close).toBe("function"); + }); +}); diff --git a/packages/plugins/sandbox-providers/daytona/src/login-pty.ts b/packages/plugins/sandbox-providers/daytona/src/login-pty.ts new file mode 100644 index 00000000000..803cafc729d --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/login-pty.ts @@ -0,0 +1,586 @@ +// The Daytona pseudo-terminal (PTY) session for the shared login flow. The login +// command needs a real pseudo-terminal: pipe stdio emits no login prompt. The +// Daytona SDK opens a PTY through `process.createPty`. This module binds that PTY +// to a small session that the login PTY transport consumes, so the login runner +// runs the command on a real terminal, streams the terminal output, and delivers +// the delayed browser code plus the Enter byte. +// +// Command contract: the host resolves a closed login command key and a validated +// session home, and it carries them in a launch descriptor. The descriptor holds +// no command string. This module maps the key to a compile-time command, so a +// caller cannot select or override the command. For the Codex key the module +// composes `exec env CODEX_HOME= `. For the +// Claude key it composes `exec ` with no CODEX_HOME. The +// module encodes the dynamic home with a POSIX shell-argument encoder, so the +// home cannot add a second shell token or a second command. +// +// Session home: the module revalidates the descriptor and the home shape, then +// creates the exact UUID directory as a NEW directory with mode 0700, owned by the +// login user. The inspection and the creation open the filesystem root, then walk +// each path component with a no-follow, directory-only open, so a symlink at any +// ancestor (the login root or a directory above it) fails closed. A single +// composite `stat` or `mkdir` follows a symlink at an ancestor; the per-component +// walk does not. The creation creates the login root if the root is absent and the +// UUID directory as a NEW directory, both relative to an open trusted descriptor. +// The module rejects a pre-existing target, a symlink, a non-directory, a wrong +// owner, and a wrong mode. It uses no recursive delete. It re-checks the directory +// with a no-symlink walk before it opens the terminal and again before it sends the +// launch input, so a symlink swapped in after creation fails before the launch. +// +// Dependency boundary: this provider plugin ships standalone (the workspace +// excludes `packages/plugins/sandbox-providers/**`). So the module imports no +// workspace package. It declares the small session shape locally through +// {@link LoginPtySession}. The shape matches the `LoginPtySession` interface in +// `@paperclipai/adapter-utils`, so the transport factory `createLoginPtyTransport` +// accepts the session opener from this module with no adapter. The runner drives +// the transport; the transport drives this session. +// +// SDK boundary: the module holds no Daytona SDK type import. It declares the small +// subset of the SDK PTY and filesystem surfaces that it uses through +// {@link DaytonaPtyProcess}, {@link DaytonaPtyHandle}, and {@link DaytonaSandboxExec}. +// The SDK `Process` and `PtyHandle` satisfy those subsets, so a caller passes +// `sandbox.process` with no adapter. The narrow surface keeps the module +// unit-testable with a fake PTY and a fake filesystem. +// +// Security (secret handling): the login runner delivers the browser code and the +// Enter byte through {@link DaytonaPtyHandle.sendInput}. The SDK sends the input +// over the PTY socket, so the code never rides a command line and never reaches a +// process argument list. The session forwards the raw terminal bytes with no ANSI +// or OSC 8 handling; the login parser owns that handling. + +import { randomUUID } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { sendPtyInputInChunks } from "./pty-chunked-input.js"; + +/** + * The closed set of login command identities. The worker maps each key to a + * compile-time command. A value outside this set fails closed before the module + * touches the filesystem. + */ +export type LoginCommandKey = "claude" | "codex"; + +/** + * The host-resolved launch descriptor. It carries the closed command key and the + * server-controlled session home. It holds no command string. + */ +export interface LoginPtyLaunchDescriptor { + /** The host-resolved fixed command identity. */ + loginCommandKey: LoginCommandKey; + /** + * The server-controlled session home. The shape is exact: + * `/tmp/paperclip-adapter-login/`. + */ + sessionHome: string; +} + +/** + * The compile-time command for each login command key. The Daytona plugin ships + * standalone, so it holds its own copy of the fixed commands. A future change to + * a command lands here and in the adapter constant together. + */ +const LOGIN_COMMAND_BY_KEY: Readonly> = { + claude: "claude setup-token", + codex: "codex login --device-auth", +}; + +/** The octal mode string for a new session home directory. */ +const LOGIN_HOME_MODE = "700"; + +/** The fixed root for a login session home. */ +const LOGIN_SESSION_HOME_ROOT = "/tmp/paperclip-adapter-login"; + +/** + * The exact absolute path shape for a login session home. The path is the fixed + * root, one slash, and one UUID. The anchors reject a relative path, a traversal, + * whitespace, a control character, and a shell metacharacter, because none of + * those match the UUID or the fixed root. + */ +const SESSION_HOME_PATTERN = + /^\/tmp\/paperclip-adapter-login\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +/** The fixed non-secret error a rejected launch descriptor or session home returns. */ +export const LOGIN_PTY_DESCRIPTOR_REJECTED = "LOGIN_PTY_DESCRIPTOR_REJECTED"; +/** The fixed non-secret error a rejected session home directory returns. */ +export const LOGIN_PTY_HOME_REJECTED = "LOGIN_PTY_HOME_REJECTED"; + +/** + * Encodes one value as a single POSIX shell argument. It wraps the value in + * single quotes and escapes each embedded single quote. The shell treats every + * other character inside single quotes as a literal, so an encoded value cannot + * add a second shell token or a second command. + */ +export function encodePosixShellArg(value: string): string { + return `'${value.split("'").join("'\\''")}'`; +} + +/** Reports whether a value is a member of the closed login command key set. */ +export function isLoginCommandKey(value: unknown): value is LoginCommandKey { + return value === "claude" || value === "codex"; +} + +/** + * Composes the launch line for the descriptor. For the Codex key it prefixes one + * safely encoded `CODEX_HOME` assignment with `env`. For the Claude key it adds no + * `CODEX_HOME`. It replaces the interactive shell with the command through `exec`, + * so the pseudo-terminal runs the command directly and its exit code becomes the + * PTY exit code. + */ +export function composeLaunchLine(descriptor: LoginPtyLaunchDescriptor): string { + const command = LOGIN_COMMAND_BY_KEY[descriptor.loginCommandKey]; + if (descriptor.loginCommandKey === "codex") { + const encodedHome = encodePosixShellArg(descriptor.sessionHome); + return `exec env CODEX_HOME=${encodedHome} ${command}`; + } + return `exec ${command}`; +} + +/** + * A live pseudo-terminal session for one login command. The session allocates a + * real pseudo-terminal, streams the raw terminal output, accepts delayed input, + * and stops the child. The shape matches the `LoginPtySession` interface in + * `@paperclipai/adapter-utils`, so the transport factory there accepts a session + * opener that returns this session. + */ +export interface LoginPtySession { + /** Registers the one output listener. The session streams each raw chunk in order. */ + onData(listener: (chunk: string) => void): void; + /** Writes raw input bytes to the pseudo-terminal. */ + write(data: string): void; + /** Resolves with the child exit code when the command ends. */ + wait(): Promise<{ exitCode: number | null }>; + /** Stops the child process. Safe to call more than one time. */ + kill(): void; + /** Releases the session resources. Safe to call more than one time. */ + close(): Promise; +} + +/** Opens a {@link LoginPtySession} for `descriptor`. The transport calls it one time. */ +export type LoginPtySessionOpener = ( + descriptor: LoginPtyLaunchDescriptor, +) => Promise; + +/** + * The subset of the Daytona SDK `PtyHandle` that the session uses. The SDK + * `PtyHandle` satisfies this interface, so a caller passes the real handle with + * no adapter. + */ +export interface DaytonaPtyHandle { + /** Resolves when the PTY socket connection is ready for input and output. */ + waitForConnection(): Promise; + /** Sends raw input bytes to the pseudo-terminal. */ + sendInput(data: string | Uint8Array): Promise; + /** Resolves with the exit result when the pseudo-terminal process ends. */ + wait(): Promise<{ exitCode?: number; error?: string }>; + /** Kills the pseudo-terminal process. */ + kill(): Promise; + /** Closes the PTY socket and releases the resources. */ + disconnect(): Promise; +} + +/** + * The options the session passes to `createPty`. The session sets a fixed + * terminal size and one output callback. It sets the working directory when the + * caller provides one. + */ +export interface DaytonaPtyCreateOptions { + id: string; + cwd?: string; + cols: number; + rows: number; + onData: (data: Uint8Array) => void | Promise; +} + +/** + * The subset of the Daytona SDK `Process` that the session uses. The SDK + * `Process` satisfies this interface, so a caller passes `sandbox.process`. + */ +export interface DaytonaPtyProcess { + createPty(options: DaytonaPtyCreateOptions): Promise; +} + +/** The result of one command run on the sandbox. */ +export interface DaytonaExecResult { + exitCode?: number; + result?: string; +} + +/** + * The subset of the Daytona SDK `Process` that the filesystem helper uses. The + * SDK `Process` satisfies this interface through `executeCommand`, so a caller + * passes `sandbox.process`. + */ +export interface DaytonaSandboxExec { + executeCommand( + command: string, + cwd?: string, + env?: Record, + timeout?: number, + ): Promise; +} + +/** One no-follow inspection of a filesystem path. */ +export interface LoginHomeInspection { + /** True when the path exists (as any type). */ + exists: boolean; + /** True when the path itself is a symbolic link. */ + isSymlink: boolean; + /** True when the path itself is a directory. */ + isDirectory: boolean; + /** The octal permission string, for example `700`. Empty when the path is absent. */ + mode: string; + /** The owner user id, or null when the path is absent. */ + ownerUid: number | null; +} + +/** + * The narrow filesystem surface the session home operations need. The production + * surface runs commands on the sandbox; a unit test injects a fake. The inspection + * and the creation walk each path component with a no-follow open, so a symlink at + * any ancestor fails closed and a symlink at the target reports the link. + */ +export interface DaytonaLoginHomeFs { + /** Resolves the user id of the login user that runs the pseudo-terminal. */ + loginUserId(): Promise; + /** + * Inspects `path` without following a symlink at the target or at any ancestor. + * It rejects (throws) when an ancestor is a symlink or a non-directory. + */ + inspect(path: string): Promise; + /** + * Creates `path` as a NEW directory with the octal `mode`. It creates the login + * root ancestor if the root is absent. It fails when the target path exists or + * when an ancestor is a symlink. It follows no composite pathname. + */ + createDirectory(path: string, mode: string): Promise; +} + +/** The options for the Daytona login PTY session. */ +export interface DaytonaLoginPtyOptions { + /** The working directory for the login PTY. Defaults to the sandbox default. */ + cwd?: string; +} + +// The terminal size for the login PTY. The login UI prints a short prompt, so a +// standard terminal size is enough. A fixed size keeps the output stable. +const LOGIN_PTY_COLS = 120; +const LOGIN_PTY_ROWS = 30; + +/** + * The input terminator that replaces the interactive shell with the login + * command. The `exec` builtin replaces the shell process image with the command, + * so the PTY runs the command directly. When the command ends, the PTY ends with + * the command exit code, and the delayed input reaches the command, not a shell. + * The terminal reads the carriage return as the Enter key. + */ +const PTY_COMMAND_TERMINATOR = "\r"; + +/** Normalizes an octal permission string to its numeric value for comparison. */ +function octalMode(value: string): number { + return Number.parseInt(value, 8); +} + +/** + * Revalidates the launch descriptor. It fails closed when the command key is + * outside the closed set or the session home shape is wrong. A unit test that + * bypasses the host boundary hits this check, so the provider never trusts an + * unvalidated descriptor. + */ +function assertDescriptor(descriptor: LoginPtyLaunchDescriptor): void { + if (!isLoginCommandKey(descriptor.loginCommandKey)) { + throw new Error(LOGIN_PTY_DESCRIPTOR_REJECTED); + } + if (!SESSION_HOME_PATTERN.test(descriptor.sessionHome)) { + throw new Error(LOGIN_PTY_DESCRIPTOR_REJECTED); + } +} + +/** + * Creates and validates the exact session home directory. It rejects a + * pre-existing target (including a symlink), creates the directory as a NEW + * directory with mode 0700, then verifies the directory type, the no-symlink + * state, the mode, and the login-user ownership. It uses no recursive delete. + */ +async function prepareSessionHome(fs: DaytonaLoginHomeFs, sessionHome: string): Promise { + const loginUid = await fs.loginUserId(); + + // Reject a pre-existing target. A no-follow inspection reports a symlink, a + // file, or a directory as present, so any pre-existing target fails here. + const before = await fs.inspect(sessionHome); + if (before.exists) { + throw new Error(LOGIN_PTY_HOME_REJECTED); + } + + // Create the exact directory as a NEW directory with mode 0700. The create + // fails when the path exists, so the create never follows a symlink. + await fs.createDirectory(sessionHome, LOGIN_HOME_MODE); + + // Verify the created directory. Reject a symlink, a non-directory, a wrong + // owner, and a wrong mode. Do not delete on a rejection; the sandbox is + // ephemeral and the provider uses no recursive delete. + const after = await fs.inspect(sessionHome); + if ( + !after.exists || + after.isSymlink || + !after.isDirectory || + after.ownerUid !== loginUid || + octalMode(after.mode) !== octalMode(LOGIN_HOME_MODE) + ) { + throw new Error(LOGIN_PTY_HOME_REJECTED); + } +} + +/** + * Re-checks the directory with a no-symlink check. It rejects a symlink and a + * non-directory, so a symlink swapped in after creation fails before the launch. + */ +async function assertHomeStillSafe(fs: DaytonaLoginHomeFs, sessionHome: string): Promise { + const now = await fs.inspect(sessionHome); + if (!now.exists || now.isSymlink || !now.isDirectory) { + throw new Error(LOGIN_PTY_HOME_REJECTED); + } +} + +/** + * Opens a Daytona PTY session for `descriptor` and returns it as a + * {@link LoginPtySession}. The function revalidates the descriptor and the home + * shape, creates and validates the session home, opens a real pseudo-terminal, + * re-checks the directory before the launch, composes the safe launch line, and + * sends it. The session streams the raw terminal output, delivers delayed input, + * and stops the child. + * + * The function decodes the terminal bytes as a UTF-8 stream, so a multibyte + * character that splits across two output chunks stays whole. It buffers the + * output until the transport registers the listener, so no early chunk is lost. + */ +export async function openDaytonaLoginPtySession( + process: DaytonaPtyProcess, + fs: DaytonaLoginHomeFs, + descriptor: LoginPtyLaunchDescriptor, + options?: DaytonaLoginPtyOptions, +): Promise { + assertDescriptor(descriptor); + // Create and validate the session home before the terminal opens. + await prepareSessionHome(fs, descriptor.sessionHome); + // Re-check the directory with a no-symlink check before `createPty`. + await assertHomeStillSafe(fs, descriptor.sessionHome); + + const decoder = new TextDecoder("utf-8"); + let listener: ((chunk: string) => void) | null = null; + let buffered = ""; + + const handle = await process.createPty({ + id: `paperclip-login-pty-${randomUUID()}`, + ...(options?.cwd ? { cwd: options.cwd } : {}), + cols: LOGIN_PTY_COLS, + rows: LOGIN_PTY_ROWS, + onData: (data: Uint8Array): void => { + // Decode the terminal bytes as a stream, so a split multibyte character + // stays whole across two chunks. Forward the raw text; the parser owns the + // ANSI and OSC 8 handling. + const text = decoder.decode(data, { stream: true }); + if (text.length === 0) return; + if (listener) listener(text); + else buffered += text; + }, + }); + + await handle.waitForConnection(); + // Re-check the directory with a no-symlink check immediately before the launch + // input, so a symlink swapped in after creation fails before `sendInput`. + await assertHomeStillSafe(fs, descriptor.sessionHome); + // Replace the interactive shell with the login command, so the pseudo-terminal + // runs the command directly. The runner then writes the delayed browser code + // to the command, not to a shell. + await handle.sendInput(composeLaunchLine(descriptor) + PTY_COMMAND_TERMINATOR); + + // The tail of the write chain. The chunker awaits each chunk, so one write can + // suspend between its chunks. The chain runs each write after the previous + // write ends, so the chunks of two writes never interleave on the wire. + let writeChain: Promise = Promise.resolve(); + + return { + onData(next: (chunk: string) => void): void { + listener = next; + if (buffered.length > 0) { + const pending = buffered; + buffered = ""; + next(pending); + } + }, + write(data: string): void { + // Send the input as byte-bounded chunks under the provider message cap. The + // login input is short today, so the latent defect never fires, but the same + // unbounded write goes through the one chunker. The chain runs this write + // after the previous write ends, so two writes keep their order and never + // interleave their chunks. A write error must not throw into the runner, so + // the runner's fixed status stays the single result path. + writeChain = writeChain.then(() => + sendPtyInputInChunks((chunk) => handle.sendInput(chunk), data), + ); + }, + async wait(): Promise<{ exitCode: number | null }> { + const result = await handle.wait(); + return { exitCode: typeof result.exitCode === "number" ? result.exitCode : null }; + }, + kill(): void { + void handle.kill().catch(() => undefined); + }, + async close(): Promise { + await handle.disconnect().catch(() => undefined); + }, + }; +} + +/** + * The inspection helper source. The provider reads it from its own script file and + * runs it on the sandbox as ` && node -e