diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 042c52bdb06..350cece37b6 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -18,7 +18,9 @@ use crate::provider_failure::validated_provider_failure; use crate::stable_identity::{is_stable_id, DURABLE_STABLE_ID_CHARS, SHORT_STABLE_ID_CHARS}; pub const ACPX_SIDECAR_MAX_FRAME_BYTES: usize = 1024 * 1024; -const MAX_BUFFERED_EVENTS: usize = 512; +const SIDECAR_MAX_UNFLUSHED_OUTPUT_BYTES: usize = 4 * ACPX_SIDECAR_MAX_FRAME_BYTES; +const MAX_BUFFERED_EVENT_BYTES: usize = 4 * SIDECAR_MAX_UNFLUSHED_OUTPUT_BYTES; +const MAX_BUFFERED_EVENTS: usize = MAX_BUFFERED_EVENT_BYTES / 256; const MAX_EVENT_POLL_TIMEOUT: Duration = Duration::from_secs(120); const MAX_JSON_SAFE_INTEGER: u64 = 9_007_199_254_740_991; /// A response wait blocks in slices of at most this long. @@ -128,7 +130,9 @@ pub struct AcpxSidecarTransport { request_deadline: Option, next_request_id: u64, last_event_sequence: u64, - buffered_events: VecDeque, + buffered_events: VecDeque<(AcpxSidecarEvent, usize)>, + buffered_event_bytes: usize, + last_polled_event_bytes: usize, stderr_tail: BoundedLogBuffer, stderr_categories: BTreeSet<&'static str>, poisoned: bool, @@ -439,6 +443,8 @@ impl AcpxSidecarTransport { next_request_id: 1, last_event_sequence: 0, buffered_events: VecDeque::new(), + buffered_event_bytes: 0, + last_polled_event_bytes: 0, stderr_tail: BoundedLogBuffer::new(32, 8 * 1024), stderr_categories: BTreeSet::new(), poisoned: false, @@ -464,7 +470,7 @@ impl AcpxSidecarTransport { } pub(crate) fn buffered_events(&self) -> impl ExactSizeIterator { - self.buffered_events.iter() + self.buffered_events.iter().map(|(event, _)| event) } pub fn request( @@ -550,22 +556,33 @@ impl AcpxSidecarTransport { "ACPX sidecar event poll timeout must not exceed 120 s", )); } - if let Some(event) = self.buffered_events.pop_front() { + self.last_polled_event_bytes = 0; + if let Some((event, bytes)) = self.buffered_events.pop_front() { + self.buffered_event_bytes -= bytes; + self.last_polled_event_bytes = bytes; return Ok(Some(event)); } if timeout.is_zero() { return Ok(None); } - let result = self.poll_event_inner(timeout); - if result.is_err() { - self.poison(); + match self.poll_event_inner(timeout) { + Ok(Some((event, bytes))) => { + self.last_polled_event_bytes = bytes; + Ok(Some(event)) + } + Ok(None) => Ok(None), + Err(error) => { + self.poison(); + Err(error) + } } - result } pub(crate) fn defer_polled_event(&mut self, event: AcpxSidecarEvent) { // poll_event already validated its sequence and freed its FIFO slot. - self.buffered_events.push_front(event); + let bytes = std::mem::take(&mut self.last_polled_event_bytes); + self.buffered_event_bytes += bytes; + self.buffered_events.push_front((event, bytes)); } pub fn shutdown(&mut self) -> Result<(), LocalRunnerError> { @@ -767,7 +784,7 @@ impl AcpxSidecarTransport { continue; }; match parse_frame(&line)? { - ParsedFrame::Event(event) => self.buffer_event(event)?, + ParsedFrame::Event(event) => self.buffer_event(event, line.len())?, ParsedFrame::Response(response) => { if self.abandoned_observation == Some(response.id) { self.abandoned_observation = None; @@ -811,7 +828,7 @@ impl AcpxSidecarTransport { fn poll_event_inner( &mut self, timeout: Duration, - ) -> Result, LocalRunnerError> { + ) -> Result, LocalRunnerError> { let deadline = Instant::now() + timeout; loop { let Some(line) = self.receive_stdout_line( @@ -824,7 +841,7 @@ impl AcpxSidecarTransport { match parse_frame(&line)? { ParsedFrame::Event(event) => { self.validate_event_sequence(event.sequence)?; - return Ok(Some(event)); + return Ok(Some((event, line.len()))); } ParsedFrame::Response(response) if self.abandoned_observation == Some(response.id) => @@ -882,14 +899,21 @@ impl AcpxSidecarTransport { } } - fn buffer_event(&mut self, event: AcpxSidecarEvent) -> Result<(), LocalRunnerError> { - if self.buffered_events.len() >= MAX_BUFFERED_EVENTS { + fn buffer_event( + &mut self, + event: AcpxSidecarEvent, + bytes: usize, + ) -> Result<(), LocalRunnerError> { + if self.buffered_events.len() >= MAX_BUFFERED_EVENTS + || self.buffered_event_bytes.saturating_add(bytes) > MAX_BUFFERED_EVENT_BYTES + { return Err(LocalRunnerError::invalid( "ACPX sidecar exceeded the buffered event limit", )); } self.validate_event_sequence(event.sequence)?; - self.buffered_events.push_back(event); + self.buffered_event_bytes += bytes; + self.buffered_events.push_back((event, bytes)); Ok(()) } @@ -994,6 +1018,7 @@ impl AcpxSidecarTransport { } self.poisoned = true; self.buffered_events.clear(); + self.buffered_event_bytes = 0; let _ = self.process.terminate_group(); } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index 188b32c7e22..f9b9e78563b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -682,8 +682,30 @@ fn run() -> Result<(), Box> { write_json(&mut stdout, &success(id, command, &request))?; } "flood" => { - for sequence in 1..=513 { - write_event(&mut stdout, sequence)?; + for _ in 0..288 { + write_padded_event(&mut stdout, next_sequence, 64 * 1024)?; + next_sequence += 1; + } + write_json(&mut stdout, &success(id, command, &request))?; + } + "count-flood" => { + for _ in 0..=65_536 { + write_event(&mut stdout, next_sequence)?; + next_sequence += 1; + } + write_json(&mut stdout, &success(id, command, &request))?; + } + "backlog" => { + for _ in 0..2_048 { + write_event(&mut stdout, next_sequence)?; + next_sequence += 1; + } + write_json(&mut stdout, &success(id, command, &request))?; + } + "large-backlog" => { + for _ in 0..192 { + write_padded_event(&mut stdout, next_sequence, 64 * 1024)?; + next_sequence += 1; } write_json(&mut stdout, &success(id, command, &request))?; } @@ -1490,6 +1512,20 @@ fn write_event(output: &mut impl Write, sequence: u64) -> io::Result<()> { ) } +fn write_padded_event(output: &mut impl Write, sequence: u64, padding: usize) -> io::Result<()> { + write_json( + output, + &json!({ + "protocolVersion": GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION, + "sequence": sequence, + "eventType": "runtime.diagnostic", + "runId": null, + "turnId": null, + "payload": { "code": "fake_event", "message": "x".repeat(padding) }, + }), + ) +} + fn write_json(output: &mut impl Write, value: &Value) -> io::Result<()> { serde_json::to_writer(&mut *output, value)?; output.write_all(b"\n")?; diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs index 9feaae2dedb..17a4558de23 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs @@ -91,13 +91,62 @@ fn a_poisoned_transport_discards_events_buffered_before_failure() { #[test] fn bounds_events_while_waiting_for_a_response() { - let mut transport = transport("flood", Duration::from_secs(2)); + let mut transport = transport("flood", Duration::from_secs(10)); let error = transport .request(GeneratedAcpxSidecarCommand::Initialize, json!({})) .expect_err("event flood must fail"); assert!(error.to_string().contains("buffered event limit")); } +#[test] +fn bounds_tiny_events_by_count_while_waiting_for_a_response() { + let mut transport = transport("count-flood", Duration::from_secs(20)); + let error = transport + .request(GeneratedAcpxSidecarCommand::Initialize, json!({})) + .expect_err("a flood of tiny events must fail"); + assert!(error.to_string().contains("buffered event limit")); +} + +#[test] +fn buffers_a_backlog_of_small_events_before_a_response() { + let mut transport = transport("backlog", Duration::from_secs(10)); + transport + .request(GeneratedAcpxSidecarCommand::Initialize, json!({})) + .expect("a backlog within the byte budget should not fail the request"); + for expected in 1..=2_048 { + let event = transport + .poll_event(Duration::ZERO) + .expect("buffered event should parse") + .expect("buffered event should exist"); + assert_eq!(event.sequence, expected); + } + assert!(transport + .poll_event(Duration::ZERO) + .expect("drained buffer should not fail") + .is_none()); + transport.shutdown().expect("fake sidecar should stop"); +} + +#[test] +fn frees_the_byte_budget_as_buffered_events_drain() { + let mut transport = transport("large-backlog", Duration::from_secs(10)); + let mut expected = 1; + for _ in 0..2 { + transport + .request(GeneratedAcpxSidecarCommand::SessionRead, json!({})) + .expect("a drained buffer should admit the next backlog"); + for _ in 0..192 { + let event = transport + .poll_event(Duration::ZERO) + .expect("buffered event should parse") + .expect("buffered event should exist"); + assert_eq!(event.sequence, expected); + expected += 1; + } + } + transport.shutdown().expect("fake sidecar should stop"); +} + #[test] fn rejects_an_oversized_stdout_frame() { let mut transport = transport("oversized", Duration::from_secs(1));