diff --git a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md index 3b61ad38db9..24d651c0d37 100644 --- a/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md +++ b/packages/paperclip-runner/runner/DURABLE_TRANSPORT.md @@ -8,6 +8,12 @@ agent or recovery of its persisted run. On a fresh launch or restart, - With `--listen-address 0.0.0.0 --listen-path /api/runner/v1/connect/`, the Runner listens for Core. `--listen-port` defaults to `43127`. +A dialing Runner waits `--reconnect-delay-ms` (default 250) between +connection attempts and backs off after a failed one. A listening Runner +checks for an incoming connection every 10 ms (or sooner, when that delay is +smaller). It never redials, so a connection does not wait out the reconnect +delay after a disconnect or an authority rotation. + Choose one form. Both require `PAPERCLIP_RUNNER_BOOTSTRAP_TICKET`. Recovery uses the same entry point. The Runner rejects `--source-reconcile`. diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index ae1d3bb7aa4..e9c2eafc585 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -730,7 +730,8 @@ pub fn run_durable_runner_with_host_control( let (mut transport, welcome) = match connection { Ok(Some(connection)) => connection, Ok(None) => { - next_connection_attempt = Instant::now() + config.reconnect_delay; + next_connection_attempt = + Instant::now() + endpoint.retry_delay(config.reconnect_delay); continue; } Err(error) => { @@ -749,7 +750,7 @@ pub fn run_durable_runner_with_host_control( } next_connection_attempt = Instant::now() + reconnect_delay( - config.reconnect_delay, + endpoint.retry_delay(config.reconnect_delay), connect_deadline.saturating_duration_since(Instant::now()), &mut reconnect_attempt, ); @@ -1055,8 +1056,8 @@ pub fn run_durable_runner_with_host_control( .map(|credential| credential.expires_at_unix_ms), ); next_connection_attempt = Instant::now() - + config - .reconnect_delay + + endpoint + .retry_delay(config.reconnect_delay) .min(reconnect_deadline.saturating_duration_since(Instant::now())); continue; } @@ -1419,8 +1420,8 @@ pub fn run_durable_runner_with_host_control( .map(|credential| credential.expires_at_unix_ms), ); next_connection_attempt = Instant::now() - + config - .reconnect_delay + + endpoint + .retry_delay(config.reconnect_delay) .min(reconnect_deadline.saturating_duration_since(Instant::now())); } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs index 8b245375882..578419e8bb4 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/transport.rs @@ -32,6 +32,7 @@ const QUEUED_CONTROL_READ_TIMEOUT: Duration = Duration::from_millis(1); const CONNECT_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(3); const CONNECT_TOTAL_TIMEOUT: Duration = Duration::from_secs(10); const MAX_CA_BUNDLE_BYTES: u64 = 4 * 1024 * 1024; +const LISTENER_ACCEPT_INTERVAL: Duration = Duration::from_millis(10); type HmacSha256 = Hmac; @@ -252,6 +253,13 @@ impl RunnerTransportEndpoint { Ok(()) } + pub(crate) fn retry_delay(&self, configured: Duration) -> Duration { + match self { + Self::Dial(_) => configured, + Self::Listen { .. } => configured.min(LISTENER_ACCEPT_INTERVAL), + } + } + fn open( &self, max_frame_bytes: usize, @@ -2172,6 +2180,27 @@ mod tests { } } + #[test] + fn listener_checks_its_backlog_sooner_than_a_dial_reconnects() { + let configured = Duration::from_millis(250); + let listener = RunnerTransportEndpoint::Listen { + listener: TcpListener::bind("0.0.0.0:0").unwrap(), + path: "/api/runner/v1/connect/run_1".to_owned(), + }; + assert_eq!(listener.retry_delay(configured), LISTENER_ACCEPT_INTERVAL); + assert!(LISTENER_ACCEPT_INTERVAL <= Duration::from_millis(25)); + assert_eq!( + listener.retry_delay(Duration::from_millis(1)), + Duration::from_millis(1) + ); + let dial = RunnerTransportEndpoint::new( + "ws://127.0.0.1:43127/api/runner/v1/connect/run_1", + "run_1", + ) + .unwrap(); + assert_eq!(dial.retry_delay(configured), configured); + } + #[test] fn listener_path_is_exact_and_unambiguous() { validate_listener_path("/api/runner/v1/connect/run_1").unwrap(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/listener_accept.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/listener_accept.rs new file mode 100644 index 00000000000..5af4fc24bf8 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/listener_accept.rs @@ -0,0 +1,138 @@ +#![cfg(unix)] + +use std::fs; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +struct Runner { + process: Child, + directory: PathBuf, +} + +impl Drop for Runner { + fn drop(&mut self) { + let _ = self.process.kill(); + let _ = self.process.wait(); + let _ = fs::remove_dir_all(&self.directory); + } +} + +const RUN_ID: &str = "run_accept"; + +fn upgrade(stream: &mut TcpStream) -> Duration { + let started = Instant::now(); + stream + .write_all( + format!( + "GET /api/runner/v1/connect/{RUN_ID} HTTP/1.1\r\nHost: 127.0.0.1\r\n\ + Upgrade: websocket\r\nConnection: Upgrade\r\n\ + Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n" + ) + .as_bytes(), + ) + .unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut response = Vec::new(); + let mut byte = [0_u8; 1]; + while !response.ends_with(b"\r\n\r\n") { + assert_eq!( + stream.read(&mut byte).unwrap(), + 1, + "Runner closed the upgrade" + ); + response.push(byte[0]); + } + assert!( + response.starts_with(b"HTTP/1.1 101"), + "{}", + String::from_utf8_lossy(&response) + ); + started.elapsed() +} + +fn spawn_listening_runner() -> (Runner, u16, TcpStream) { + for _ in 0..10 { + let reservation = TcpListener::bind("0.0.0.0:0").unwrap(); + let port = reservation.local_addr().unwrap().port(); + drop(reservation); + let directory = + std::env::temp_dir().join(format!("runner-accept-{}", uuid::Uuid::new_v4())); + fs::create_dir(&directory).unwrap(); + fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)).unwrap(); + let process = Command::new(env!("CARGO_BIN_EXE_paperclip-runnerd")) + .env( + "PAPERCLIP_RUNNER_BOOTSTRAP_TICKET", + "fixture-bootstrap-ticket", + ) + .args([ + "--listen-address", + "0.0.0.0", + "--listen-port", + &port.to_string(), + "--listen-path", + &format!("/api/runner/v1/connect/{RUN_ID}"), + "--state-dir", + directory.to_str().unwrap(), + "--runner-id", + "runner", + "--environment-lease-id", + "lease", + "--run-id", + RUN_ID, + "--session-id", + "session", + "--turn-id", + "turn", + "--item-id", + "item", + "--runner-version", + "0.0.0", + "--runner-digest", + &format!("sha256:{}", "a".repeat(64)), + "--max-runtime-ms", + "20000", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut runner = Runner { process, directory }; + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if let Some(status) = runner.process.try_wait().unwrap() { + match TcpListener::bind(("0.0.0.0", port)) { + Err(error) if error.kind() == std::io::ErrorKind::AddrInUse => break, + _ => panic!("Runner exited before listening: {status}"), + } + } + if let Ok(stream) = TcpStream::connect(("127.0.0.1", port)) { + return (runner, port, stream); + } + assert!(Instant::now() < deadline, "Runner did not bind its port"); + thread::sleep(Duration::from_millis(10)); + } + } + panic!("could not acquire an available Runner port"); +} + +#[test] +fn listener_answers_the_next_connection_without_waiting_out_the_reconnect_delay() { + let (_runner, port, mut stale) = spawn_listening_runner(); + upgrade(&mut stale); + drop(stale); + thread::sleep(Duration::from_millis(30)); + + let mut current = TcpStream::connect(("127.0.0.1", port)).unwrap(); + let waited = upgrade(&mut current); + assert!( + waited < Duration::from_millis(120), + "listener answered after {waited:?}" + ); +}