diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs index fd211ab9a4d..cbde98cd150 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs @@ -14,6 +14,7 @@ use paperclip_runner_core::durable::{ }; use paperclip_runner_core::local_runner::{run_local_runner, LocalRunnerError, RunnerConfig}; use paperclip_runner_core::native_provider_backend::NativeProviderCommandExecutor; +use paperclip_runner_core::oom_score::{parse_oom_score_adj, set_own_oom_score_adj}; use serde_json::json; const RUNNERD_BUILD_METADATA_SCHEMA: &str = "paperclip-runner/runnerd-build-metadata/v1"; @@ -452,6 +453,20 @@ fn run_durable(args: &[String]) -> Result<(), LocalRunnerError> { .map_err(|error| LocalRunnerError::invalid(error.to_string())) } +fn apply_oom_score_adj(args: &[String]) -> Result<(), LocalRunnerError> { + let Some(value) = optional_value(args, "--oom-score-adj")? else { + return Ok(()); + }; + let score = parse_oom_score_adj(&value).ok_or_else(|| { + LocalRunnerError::invalid("invalid --oom-score-adj: expected -1000..=1000") + })?; + // Without the privilege, Runner is only less protected than requested. + if let Err(error) = set_own_oom_score_adj(score) { + eprintln!("paperclip-runnerd: failed to set --oom-score-adj: {error}"); + } + Ok(()) +} + fn run(args: &[String]) -> Result<(), LocalRunnerError> { #[cfg(unix)] if matches!( @@ -471,6 +486,7 @@ fn run(args: &[String]) -> Result<(), LocalRunnerError> { println!("{}", metadata_with_provider_packs(args)?); return Ok(()); } + apply_oom_score_adj(args)?; if args.iter().any(|argument| argument == "--connect-url") || args.iter().any(|argument| argument == "--listen-address") || args.iter().any(|argument| argument == "--listen-port") diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs index ffbc524010e..8b063e95c93 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs @@ -26,6 +26,7 @@ pub mod local_runner; pub mod managed_provider; pub mod managed_provider_backend; pub mod native_provider_backend; +pub mod oom_score; pub mod process_supervisor; pub mod provider_backend; pub mod provider_bridge; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/oom_score.rs b/packages/paperclip-runner/runner/crates/runner-core/src/oom_score.rs new file mode 100644 index 00000000000..f0448aa5dd5 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/oom_score.rs @@ -0,0 +1,105 @@ +//! Linux out-of-memory priority for Runner and the processes it supervises. + +use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; + +const SCORE_RANGE: std::ops::RangeInclusive = -1000..=1000; + +// Set only by the launch flag, so a launcher that does not opt in keeps today's +// inheritance even when Runner starts with a negative score. +static RELEASE_SUPERVISED_PROCESSES: AtomicBool = AtomicBool::new(false); + +pub fn parse_oom_score_adj(value: &str) -> Option { + value + .parse() + .ok() + .filter(|score| SCORE_RANGE.contains(score)) +} + +/// Each environment decides how much Runner is worth protecting, so Runner has +/// no default. A negative value needs privilege or a floor set by the launcher; +/// callers treat a failure as non-fatal because it only leaves Runner killable. +pub fn set_own_oom_score_adj(score: i16) -> io::Result<()> { + // The launcher opted in even if this write fails and Runner keeps its score. + RELEASE_SUPERVISED_PROCESSES.store(true, Ordering::Relaxed); + #[cfg(target_os = "linux")] + { + std::fs::write("/proc/self/oom_score_adj", score.to_string()) + } + #[cfg(not(target_os = "linux"))] + { + let _ = score; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "oom_score_adj is supported only on Linux", + )) + } +} + +/// Children inherit Runner's score. With the launch flag set, protection meant for +/// Runner must not make harness and tool processes as hard to kill as Runner. +/// +/// A fork in the window after exec keeps Runner's score. So does the child if this +/// write fails, which leaves it no more protected than Runner, so it is not fatal. +pub(crate) fn release_supervised_process(pid: u32) { + if !RELEASE_SUPERVISED_PROCESSES.load(Ordering::Relaxed) { + return; + } + #[cfg(target_os = "linux")] + if let Some(score) = std::fs::read_to_string("/proc/self/oom_score_adj") + .ok() + .and_then(|score| score.trim().parse().ok()) + { + release_from_score(score, pid); + } + #[cfg(not(target_os = "linux"))] + let _ = pid; +} + +#[cfg(target_os = "linux")] +fn release_from_score(own_score: i16, pid: u32) { + if own_score < 0 { + let _ = std::fs::write(format!("/proc/{pid}/oom_score_adj"), "0"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_only_the_kernel_range() { + assert_eq!(parse_oom_score_adj("-1000"), Some(-1000)); + assert_eq!(parse_oom_score_adj("1000"), Some(1000)); + assert_eq!(parse_oom_score_adj("-1001"), None); + assert_eq!(parse_oom_score_adj("1001"), None); + assert_eq!(parse_oom_score_adj("low"), None); + } + + #[test] + #[cfg(target_os = "linux")] + fn releases_children_only_from_a_protected_score() { + struct Sleeper(std::process::Child); + impl Drop for Sleeper { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + let child = Sleeper( + std::process::Command::new("sleep") + .arg("30") + .spawn() + .unwrap(), + ); + let pid = child.0.id(); + let path = format!("/proc/{pid}/oom_score_adj"); + let score = || std::fs::read_to_string(&path).unwrap().trim().to_owned(); + std::fs::write(&path, "7").unwrap(); + + release_from_score(0, pid); + assert_eq!(score(), "7"); + release_from_score(-1, pid); + assert_eq!(score(), "0"); + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs index 437f44f4b69..93a45779118 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs @@ -460,7 +460,7 @@ impl VerifiedProcessLaunch { environment: &std::collections::BTreeMap, ) -> Result { let inherited = self.inherited_command()?; - Command::new(&inherited.program) + let child = Command::new(&inherited.program) .args(&inherited.args) .env_clear() .envs(environment) @@ -468,7 +468,9 @@ impl VerifiedProcessLaunch { .stdout(Stdio::piped()) .stderr(Stdio::inherit()) .spawn() - .map_err(|_| LocalRunnerError::invalid("failed to spawn verified stdio MCP server")) + .map_err(|_| LocalRunnerError::invalid("failed to spawn verified stdio MCP server"))?; + crate::oom_score::release_supervised_process(child.id()); + Ok(child) } pub fn with_inherited_runtime_executable(mut self) -> Self { @@ -1077,6 +1079,7 @@ impl SupervisedProcess { )) })?; let process_group_id = child.id(); + crate::oom_score::release_supervised_process(process_group_id); let stdin = child .stdin .take() diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/oom_score.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/oom_score.rs new file mode 100644 index 00000000000..d8a021decaf --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/oom_score.rs @@ -0,0 +1,148 @@ +#![cfg(target_os = "linux")] + +use std::fs; +use std::net::TcpListener; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use paperclip_runner_core::oom_score::set_own_oom_score_adj; +use paperclip_runner_core::process_supervisor::SupervisedProcess; + +fn listener_runner(directory: &Path, port: u16, flag: &[&str]) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_paperclip-runnerd")); + command + .env( + "PAPERCLIP_RUNNER_BOOTSTRAP_TICKET", + "fixture-bootstrap-ticket", + ) + .args(flag) + .args(["--listen-address", "0.0.0.0", "--listen-port"]) + .arg(port.to_string()) + .args(["--listen-path", "/api/runner/v1/connect/run", "--state-dir"]) + .arg(directory) + .args([ + "--runner-id", + "runner", + "--environment-lease-id", + "lease", + "--run-id", + "run", + "--session-id", + "session", + "--turn-id", + "turn", + "--item-id", + "item", + "--runner-version", + "0.0.0", + "--runner-digest", + &format!("sha256:{}", "a".repeat(64)), + "--max-runtime-ms", + "10000", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + command +} + +fn private_directory() -> std::path::PathBuf { + let directory = std::env::temp_dir().join(format!("runner-oom-{}", uuid::Uuid::new_v4())); + fs::create_dir(&directory).unwrap(); + fs::set_permissions(&directory, fs::Permissions::from_mode(0o700)).unwrap(); + directory +} + +#[test] +fn launch_flag_sets_the_runner_score() { + let directory = private_directory(); + let reservation = TcpListener::bind("0.0.0.0:0").unwrap(); + let port = reservation.local_addr().unwrap().port(); + drop(reservation); + let mut runner = listener_runner(&directory, port, &["--oom-score-adj", "9"]) + .spawn() + .unwrap(); + let path = format!("/proc/{}/oom_score_adj", runner.id()); + let deadline = Instant::now() + Duration::from_secs(10); + let mut score = String::new(); + while Instant::now() < deadline { + score = fs::read_to_string(&path).unwrap(); + if score.trim() == "9" { + break; + } + thread::sleep(Duration::from_millis(10)); + } + let _ = runner.kill(); + let _ = runner.wait(); + let _ = fs::remove_dir_all(&directory); + assert_eq!(score.trim(), "9"); +} + +// Lowering is refused without privilege, and that must not stop the run. +#[test] +fn launch_flag_starts_without_the_privilege_to_apply_it() { + let directory = private_directory(); + let reservation = TcpListener::bind("0.0.0.0:0").unwrap(); + let port = reservation.local_addr().unwrap().port(); + drop(reservation); + let mut runner = listener_runner(&directory, port, &["--oom-score-adj", "-1"]) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + thread::sleep(Duration::from_millis(500)); + let status = runner.try_wait().unwrap(); + let _ = runner.kill(); + let _ = runner.wait(); + let _ = fs::remove_dir_all(&directory); + assert_eq!(status, None); +} + +#[test] +fn launch_flag_rejects_an_invalid_score() { + let directory = private_directory(); + for flag in [ + &["--oom-score-adj", "low"][..], + &["--oom-score-adj", "1001"][..], + &["--oom-score-adj"][..], + ] { + let output = listener_runner(&directory, 43127, flag).output().unwrap(); + assert!(!output.status.success()); + } + let _ = fs::remove_dir_all(&directory); +} + +// Runs under CAP_SYS_RESOURCE only: nothing else in this binary reads its score. +#[test] +fn supervised_processes_do_not_inherit_protection() { + if set_own_oom_score_adj(-1).is_err() { + eprintln!("skipped: lowering oom_score_adj requires CAP_SYS_RESOURCE"); + return; + } + let directory = private_directory(); + let result = directory.join("score"); + let mut process = SupervisedProcess::spawn( + Path::new("/bin/sh"), + &[ + "-c".to_owned(), + format!( + "read -r start; cat /proc/self/oom_score_adj > {}.tmp; mv {0}.tmp {0}; read -r finish", + result.display() + ), + ], + Duration::from_secs(2), + 1024, + ) + .unwrap(); + // Read the score only after spawn returns, which is when Runner has reset it. + process.send(&"start").unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !result.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let score = fs::read_to_string(&result).unwrap(); + let _ = set_own_oom_score_adj(0); + let _ = fs::remove_dir_all(&directory); + assert_eq!(score.trim(), "0"); +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/oom_score_without_flag.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/oom_score_without_flag.rs new file mode 100644 index 00000000000..a394e2789bb --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/oom_score_without_flag.rs @@ -0,0 +1,44 @@ +#![cfg(target_os = "linux")] + +use std::fs; +use std::path::Path; +use std::thread; +use std::time::{Duration, Instant}; + +use paperclip_runner_core::process_supervisor::SupervisedProcess; + +// A separate test binary: the flag's process-wide opt-in must stay unset here. +// Runs under CAP_SYS_RESOURCE only, because lowering a score needs it. +#[test] +fn supervised_processes_keep_inheritance_without_the_flag() { + if fs::write("/proc/self/oom_score_adj", "-1").is_err() { + eprintln!("skipped: lowering oom_score_adj requires CAP_SYS_RESOURCE"); + return; + } + let directory = std::env::temp_dir().join(format!("runner-oom-{}", uuid::Uuid::new_v4())); + fs::create_dir(&directory).unwrap(); + let result = directory.join("score"); + let mut process = SupervisedProcess::spawn( + Path::new("/bin/sh"), + &[ + "-c".to_owned(), + format!( + "read -r start; cat /proc/self/oom_score_adj > {}.tmp; mv {0}.tmp {0}; read -r finish", + result.display() + ), + ], + Duration::from_secs(2), + 1024, + ) + .unwrap(); + // Read the score only after spawn returns, when a reset would have happened. + process.send(&"start").unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !result.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let score = fs::read_to_string(&result).unwrap(); + let _ = fs::write("/proc/self/oom_score_adj", "0"); + let _ = fs::remove_dir_all(&directory); + assert_eq!(score.trim(), "-1"); +}